From 625b3e10d8508d222adb705a1efd6899464f7ecb Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 21:25:25 +0800 Subject: [PATCH] docs(pchronicle): sync CLI help with catalog mounts and manifests Reflect mount-all --catalog-config, dataset add|remove|list, s3:// alias region fallback, and chronicle.manifest discovery in clap help and en/zh guides. Co-authored-by: Cursor --- crates/persisting-pchronicle-cli/src/lib.rs | 27 +++++-- crates/persisting-pchronicle-cli/src/tests.rs | 13 ++++ docs/src/en/pchronicle/design/architecture.md | 15 ++-- docs/src/en/pchronicle/design/catalog.md | 8 ++ docs/src/en/pchronicle/guides/exchange.md | 6 +- docs/src/en/pchronicle/guides/serve.md | 33 ++++++-- docs/src/en/pchronicle/guides/ui.md | 12 +-- .../en/pchronicle/reference/cases-platform.md | 75 ++++++++++--------- docs/src/en/pchronicle/reference/cli.md | 52 +++++++------ docs/src/zh/pchronicle/design/architecture.md | 11 ++- docs/src/zh/pchronicle/design/catalog.md | 6 ++ docs/src/zh/pchronicle/guides/exchange.md | 5 +- docs/src/zh/pchronicle/guides/serve.md | 32 ++++++-- docs/src/zh/pchronicle/guides/ui.md | 8 +- .../zh/pchronicle/reference/cases-platform.md | 65 ++++++++-------- docs/src/zh/pchronicle/reference/cli.md | 43 ++++++----- 16 files changed, 261 insertions(+), 150 deletions(-) diff --git a/crates/persisting-pchronicle-cli/src/lib.rs b/crates/persisting-pchronicle-cli/src/lib.rs index 63275b42..75430b4b 100644 --- a/crates/persisting-pchronicle-cli/src/lib.rs +++ b/crates/persisting-pchronicle-cli/src/lib.rs @@ -356,8 +356,11 @@ S3-compatible endpoints can be stored separately with `--endpoint URL` and are applied as AWS_ENDPOINT_URL_S3 when the alias is used. HTTP endpoints automatically enable AWS_ALLOW_HTTP for local S3-compatible services such as MinIO. -An optional `--region REGION` is stored per alias; when omitted, the client -falls back to `us-west-2` only when it needs a region. +An optional `--region REGION` is stored per alias; when omitted for an +`s3://` alias, pChronicle applies `AWS_REGION` / `AWS_DEFAULT_REGION` as +`us-west-2` before opening the store (OpenDAL requires a region). +Endpoint, region, and credentials are applied before the Tokio runtime starts +so local MinIO-style endpoints work without exporting AWS_* in the shell. A `catalog://127.0.0.1:PORT` alias is a Directory locator: `@team/prod` fetches a ticket and opens the ticket path. User `--ak/--sk` are required; `--endpoint` and `--region` are not accepted. Backend object-store keys stay on @@ -386,7 +389,7 @@ enum AliasCommand { /// S3-compatible service endpoint, stored separately from the Dataset URI. #[arg(long, value_name = "URL")] endpoint: Option, - /// S3 region. If omitted, the client default (`us-west-2`) is used when needed. + /// S3 region. If omitted for s3:// aliases, defaults to us-west-2. #[arg(long, value_name = "REGION")] region: Option, /// S3 access key ID. Must be provided together with --sk. @@ -420,7 +423,7 @@ enum AliasCommand { /// Replace the S3-compatible service endpoint stored for this alias. #[arg(long, value_name = "URL")] endpoint: Option, - /// Replace the S3 region stored for this alias. + /// Replace the S3 region stored for this alias (omit to keep; clear by re-adding). #[arg(long, value_name = "REGION")] region: Option, /// Replace the S3 access key ID stored for this alias. @@ -904,7 +907,7 @@ struct ExportArgs { ) )] struct ServeArgs { - /// Issue catalog users or change grants without starting Warehouse. + /// Manage Directory ACL or start Warehouse without a catalog subcommand. #[command(subcommand)] command: Option, @@ -994,8 +997,10 @@ struct ServeArgs { #[arg(long = "gateway-debug", alias = "debug", requires = "gateway_config")] debug: bool, - /// Directory ACL file (libraries + users). Enables catalog:// locators and - /// per-user query workers for the Web API. + /// Directory ACL file (libraries + users). Mounts every [datasets.*] entry + /// into Warehouse and enables catalog:// locators. Mutually exclusive with + /// positional Dataset mounts. Apply S3 endpoint/region/keys from the file + /// before opening stores. #[arg( long = "catalog-config", value_name = "FILE", @@ -1010,7 +1015,7 @@ struct ServeArgs { #[derive(Debug, Subcommand)] enum ServeSubcommand { - /// Issue catalog users and grant libraries without starting HTTP. + /// Manage Directory ACL (users, grants, datasets) without starting HTTP. Catalog(CatalogManageArgs), } @@ -1059,16 +1064,22 @@ enum CatalogDatasetCommand { struct CatalogDatasetAddArgs { #[command(flatten)] file: CatalogFileArg, + /// Library / mount name (becomes the Warehouse dataset name). #[arg(value_name = "NAME")] name: String, + /// Dataset URI (local path or s3://bucket/prefix). #[arg(long = "uri", value_name = "URI")] uri: String, + /// S3-compatible endpoint for this library (required consistency across s3:// entries). #[arg(long = "endpoint", value_name = "URL")] endpoint: Option, + /// S3 region for this library (required for s3:// when not relying on process env). #[arg(long = "region", value_name = "REGION")] region: Option, + /// Backend object-store access key (not a Directory user key). #[arg(long = "access-key", value_name = "KEY")] access_key: Option, + /// Backend object-store secret key (not a Directory user key). #[arg(long = "secret-key", value_name = "KEY")] secret_key: Option, #[arg(long, value_enum, default_value_t = OutputFormat::Auto)] diff --git a/crates/persisting-pchronicle-cli/src/tests.rs b/crates/persisting-pchronicle-cli/src/tests.rs index 9f9d3bed..06c09611 100644 --- a/crates/persisting-pchronicle-cli/src/tests.rs +++ b/crates/persisting-pchronicle-cli/src/tests.rs @@ -320,12 +320,25 @@ fn command_tree_contains_the_product_commands() { .map(|command| command.get_name()) .collect::>(); assert_eq!(catalog_commands, ["issue", "grant", "revoke", "dataset"]); + let dataset = catalog + .get_subcommands() + .find(|command| command.get_name() == "dataset") + .unwrap(); + let dataset_commands = dataset + .get_subcommands() + .map(|command| command.get_name()) + .collect::>(); + assert_eq!(dataset_commands, ["add", "remove", "list"]); let mut serve_command = Cli::command(); let serve_help = serve_command.find_subcommand_mut("serve").unwrap(); let mut help = Vec::new(); serve_help.write_long_help(&mut help).unwrap(); let help = String::from_utf8(help).unwrap(); assert!(help.contains("pchronicle serve catalog"), "{help}"); + assert!( + help.contains("Mounts every [datasets.*] entry"), + "{help}" + ); } #[test] diff --git a/docs/src/en/pchronicle/design/architecture.md b/docs/src/en/pchronicle/design/architecture.md index 2c0868a5..8e849000 100644 --- a/docs/src/en/pchronicle/design/architecture.md +++ b/docs/src/en/pchronicle/design/architecture.md @@ -149,11 +149,12 @@ Snapshot before switching readers. Dataset tables prune by Source before opening matching fixed versions; caches and routing indexes are tied to that Snapshot generation. -With `--catalog-config`, the parent process serves Directory list/ticket routes -and does not open those paths itself. Authorized Web queries run in a worker -that only receives the caller's paths. After a CLI ticket, the client opens the -ticket `uri` (a path) with storage credentials. That is platform addressing over -paths, not a new Dataset kind. +With `--catalog-config`, Warehouse mounts every `[datasets.*]` library from the +Directory ACL file (same data plane as positional mounts) and also serves +Directory list/ticket routes for `catalog://` aliases. Backend S3 endpoint, +region, and keys from the file are applied before stores open. After a CLI +ticket, the client opens the ticket `uri` (a path) with storage credentials. +That is platform addressing over paths, not a new Dataset kind. The Web application and API are consumers of the same read model. They do not become another source of truth. Unknown API routes remain errors rather than SPA @@ -178,7 +179,9 @@ Gateway composition belong to the [`pchronicle` reference](../reference/cli.md). - [Snapshot design](catalog.md): discovery, Snapshot construction, lazy Source resolution, and pruning. - [RFC-0013 path Directory](../../rfcs/0013-pchronicle-warehouse-catalog.md): - name-to-path resolution, ACL, tickets, and query workers. + name-to-path resolution, ACL, and tickets. +- [RFC-0015 `chronicle.manifest`](../../rfcs/0015-chronicle-manifest.md): nested + Dataset discovery and aggregate-stat sidecars. - [Run storage](trajectory-storage.md): canonical facts, storage layouts, and write ownership. - [Storyline Lance](storyline-lance.md): three-table projection, content layer, diff --git a/docs/src/en/pchronicle/design/catalog.md b/docs/src/en/pchronicle/design/catalog.md index b399a221..0e6cb574 100644 --- a/docs/src/en/pchronicle/design/catalog.md +++ b/docs/src/en/pchronicle/design/catalog.md @@ -242,6 +242,14 @@ Stopping descent after a composite root is recognized keeps manifests, generations, segments, and `objects.lance` from being treated as user input. +When a directory contains `chronicle.manifest` +([RFC-0015](../../rfcs/0015-chronicle-manifest.md)), discovery prefers that +sidecar: a `leaf` with `format = compact-jsonl/v1` becomes a Compact source +without opening Lance solely to classify it; a `branch` scans only immediate +child directories that also have the sidecar. Explorer folder totals may use +leaf `record_count` with read-side roll-up; writers update only the leaf +manifest and do not rewrite ancestors. + ### 5.2 Local discovery Local URIs accept ordinary paths, `local://`, and `file://`: diff --git a/docs/src/en/pchronicle/guides/exchange.md b/docs/src/en/pchronicle/guides/exchange.md index d845ebf3..34554888 100644 --- a/docs/src/en/pchronicle/guides/exchange.md +++ b/docs/src/en/pchronicle/guides/exchange.md @@ -10,7 +10,11 @@ session JSONL. Export refuses those two formats. Compact JSONL keeps one JSON object per row without assigning trajectory semantics. Use `--input-format compact-jsonl` or `--output-format compact-jsonl`; see the [CLI reference](../reference/cli.md) -for the `--column` mapping and snapshot-sync restrictions. +for the `--column` mapping and snapshot-sync restrictions. Records without a +usable `id` receive a stable `source_filename#line_number` identity; export +preserves original input bytes. Successful compact import also writes a leaf +`chronicle.manifest` at the dataset root so later discovery can avoid opening +Lance only to classify the tree ([RFC-0015](../../rfcs/0015-chronicle-manifest.md)). ## Import into a new Dataset diff --git a/docs/src/en/pchronicle/guides/serve.md b/docs/src/en/pchronicle/guides/serve.md index c1fb759d..dc0bc944 100644 --- a/docs/src/en/pchronicle/guides/serve.md +++ b/docs/src/en/pchronicle/guides/serve.md @@ -15,6 +15,9 @@ pchronicle serve [--gateway-stream-markdown] [--gateway-debug] [--catalog-config FILE] [<[NAME=]DATASET> ...] +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... +pchronicle serve catalog dataset list --catalog-config FILE pchronicle serve catalog issue --catalog-config FILE NAME pchronicle serve catalog grant --catalog-config FILE NAME DATASET... pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... @@ -49,17 +52,30 @@ still set mount names explicitly. ## Serve a path Directory ```bash +pchronicle serve catalog dataset add \ + --catalog-config catalog.toml prod \ + --uri s3://bucket/prod \ + --endpoint http://127.0.0.1:9000 \ + --region us-west-2 \ + --access-key BACKEND_AK \ + --secret-key BACKEND_SK pchronicle serve catalog issue --catalog-config catalog.toml alice pchronicle serve catalog grant --catalog-config catalog.toml alice prod evals pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 ``` -`catalog.toml` lists libraries (each a path) and users. `serve catalog issue` -writes a user with empty grants and prints the secret once on stdout; `grant` / -`revoke` change `datasets` without starting HTTP. Restart serve after editing -the file. The parent process does not open those paths itself. The Web UI sends -user access/secret keys as headers; queries run in a one-shot worker that -receives only that user's paths. From another terminal: +`catalog.toml` lists libraries (`[datasets.*]`, each a path or `s3://` URI) and +users. `serve catalog dataset add|remove|list` rewrites libraries without +starting HTTP. `serve catalog issue` writes a user with empty grants and prints +the secret once on stdout; `grant` / `revoke` change which library names that +user may open. Restart serve after editing the file. + +`pchronicle serve --catalog-config` mounts **every** library in the file into +Warehouse (same as positional mounts). It also enables Directory ticket routes +for `catalog://` aliases. Do not combine `--catalog-config` with positional +Dataset mounts. Backend S3 endpoint, region, and keys from the file are applied +before stores open. The Web UI may send Directory user access/secret keys as +headers when you use catalog-authenticated flows. From another terminal: ```bash pchronicle alias add team catalog://127.0.0.1:8081 --ak USER_AK --sk USER_SK @@ -69,8 +85,9 @@ pchronicle query @team/prod --sql 'SELECT 1' `@team` is a Directory locator, not a Dataset. `@team/prod` fetches a ticket and opens the ticket `uri` (a path). All `s3://` libraries in one Directory file must share the same endpoint, region, and backend keys. The listener remains -loopback-only. The design is specified in -[RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md). +loopback-only. Nested Dataset discovery may use `chronicle.manifest` sidecars +([RFC-0015](../../rfcs/0015-chronicle-manifest.md)). The Directory design is +specified in [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md). ## Enable Control or Gateway integration diff --git a/docs/src/en/pchronicle/guides/ui.md b/docs/src/en/pchronicle/guides/ui.md index 2ea55b7d..f1d24967 100644 --- a/docs/src/en/pchronicle/guides/ui.md +++ b/docs/src/en/pchronicle/guides/ui.md @@ -144,11 +144,13 @@ untrusted or shared browser profile. Clearing this site's browser data also clears the setting. Assistant is labeled **Read-only · selected run data** and does not rewrite the Dataset. -When `pchronicle serve --catalog-config` is used, open **Keys** on the left rail -and enter the Directory user access key and secret key. Those values are stored in -`localStorage` and sent to this pChronicle server as `x-pchronicle-access-key` -and `x-pchronicle-secret-key` on data requests. They authorize which paths this -browser may open; they are not the object-store backend keys. +When `pchronicle serve --catalog-config` is used, every library in the ACL file +is already mounted for local browsing. Open **Keys** on the left rail if you +need Directory user access/secret headers for authenticated Directory flows. +Those values are stored in `localStorage` and sent to this pchronicle server as +`x-pchronicle-access-key` and `x-pchronicle-secret-key` on data requests. They +authorize which Directory paths this browser may open; they are not the +object-store backend keys. ## Troubleshooting diff --git a/docs/src/en/pchronicle/reference/cases-platform.md b/docs/src/en/pchronicle/reference/cases-platform.md index 93c99960..e37190a0 100644 --- a/docs/src/en/pchronicle/reference/cases-platform.md +++ b/docs/src/en/pchronicle/reference/cases-platform.md @@ -1,44 +1,48 @@ -# pChronicle 集群平台与 Catalog Server 场景 +# pChronicle Directory and platform cases -本文覆盖平台化部署。Catalog 配置只管理用户、Dataset 和授权;Warehouse 的服务参数仍由 `pchronicle serve` 提供。 +Platform-oriented Directory setup. The ACL file manages users, datasets +(libraries), and grants; Warehouse listen/Gateway options still come from +`pchronicle serve`. -## P01:从空配置创建 Catalog 用户 +## P01: Issue a Directory user from an empty config ```bash -pchronicle serve catalog user create \ +pchronicle serve catalog issue \ --catalog-config ./catalog.toml alice ``` -如果文件不存在,命令创建配置文件、生成用户 AK/SK,并只在本次输出 secret。 +If the file does not exist, the command creates it, writes a user with empty +grants, and prints the secret once on stdout. -## P02:登记 Dataset +## P02: Register a dataset library ```bash -pchronicle serve catalog dataset create \ +pchronicle serve catalog dataset add \ --catalog-config ./catalog.toml \ - prod s3://bucket/prod \ + prod \ + --uri s3://bucket/prod \ --endpoint http://127.0.0.1:9000 \ --region us-west-2 \ - --ak BACKEND_AK \ - --sk BACKEND_SK + --access-key BACKEND_AK \ + --secret-key BACKEND_SK ``` -该命令只登记 Dataset,不创建或删除后端数据。 +This only registers the URI and backend credentials. It does not create or +delete object-store data. All `s3://` libraries in one file must share the same +endpoint, region, and backend keys. -## P03:授权用户 +## P03: Grant libraries to a user ```bash pchronicle serve catalog grant \ --catalog-config ./catalog.toml \ - alice prod \ - --permission read \ - --permission query \ - --permission analyze + alice prod ``` -预期:配置中出现独立的 `[[grants]]` 记录。 +Expected: a `[[grants]]` entry lists `prod` under that user. v1 grants are +library membership (not `--permission` flags). -## P04:启动 Catalog Server +## P04: Serve with catalog mounts ```bash pchronicle serve \ @@ -46,9 +50,10 @@ pchronicle serve \ --listen 127.0.0.1:8081 ``` -父进程负责用户认证、Dataset 列表和 ticket;查询数据面在授权 mounts 的 worker 中执行。 +Every `[datasets.*]` entry is mounted into Warehouse. Directory ticket routes +remain available for `catalog://` aliases. Restart after editing the ACL file. -## P05:访问授权 Dataset +## P05: Open an authorized dataset via Directory alias ```bash pchronicle alias add team catalog://127.0.0.1:8081 \ @@ -57,21 +62,21 @@ pchronicle query @team/prod \ --sql 'SELECT COUNT(*) AS runs FROM dataset.runs' ``` -预期:授权用户可以查询 `prod`;未授权用户或未知 Dataset 返回相同的 404 资源错误。 +Expected: an authorized user can query `prod`; unknown datasets fail closed. -## P06:撤销授权 +## P06: Revoke a library grant ```bash pchronicle serve catalog revoke \ --catalog-config ./catalog.toml \ - alice prod --permission query + alice prod ``` -预期:后续查询被拒绝,但 `read` 和其它仍保留的权限不受影响。 +Expected: later `@team/prod` access is denied for that user. -## P07:RustFS Warehouse 回归 +## P07: RustFS Warehouse regression -准备 RustFS,并设置: +Prepare RustFS and set: ```bash export PCHRONICLE_RUSTFS_ENDPOINT=http://127.0.0.1:9000 @@ -80,13 +85,15 @@ export PCHRONICLE_RUSTFS_SECRET_KEY=rustfsadmin export PCHRONICLE_RUSTFS_BUCKET=pchronicle-cases ``` -然后运行 RustFS 回归测试,验证 Dataset 写入、Catalog discovery、SQL 查询、Explorer 和 refresh 行为。 +Then run the RustFS regression coverage for Dataset writes, Snapshot discovery +(including `chronicle.manifest` when present), SQL, Explorer, and refresh. -平台验收重点: +Platform checks: -- Catalog 文件可从空文件开始构建; -- 用户、Dataset 和 grants 修改是确定性的; -- Dataset 后端凭据只在授权 ticket 中使用; -- Worker 只收到当前用户被授权的 mounts; -- Catalog refresh 不影响已完成查询的 snapshot; -- RustFS 上的 Warehouse 行为与本地 Dataset 一致。 +- ACL files can be built from empty; +- user, dataset, and grant edits are deterministic; +- backend object-store keys stay in the catalog file / ticket path, not in + `alias list` output; +- Warehouse mounts every registered library when serving `--catalog-config`; +- Snapshot refresh does not mutate an in-flight Snapshot; +- RustFS Warehouse behavior matches local Datasets for the covered paths. diff --git a/docs/src/en/pchronicle/reference/cli.md b/docs/src/en/pchronicle/reference/cli.md index df5b3d73..27fb74cd 100644 --- a/docs/src/en/pchronicle/reference/cli.md +++ b/docs/src/en/pchronicle/reference/cli.md @@ -87,7 +87,7 @@ pchronicle alias [list|add|remove|rename|get-url|set-url] [ARGUMENTS] ```bash pchronicle alias add prod s3://bucket/evals pchronicle alias add secure s3://bucket/evals --ak "$AWS_ACCESS_KEY_ID" --sk "$AWS_SECRET_ACCESS_KEY" -pchronicle alias add minio s3://bucket/evals --endpoint http://127.0.0.1:9000 --ak 123 --sk 123 +pchronicle alias add minio s3://bucket/evals --endpoint http://127.0.0.1:9000 --region us-west-2 --ak 123 --sk 123 pchronicle alias add regional s3://bucket/evals --region us-west-2 pchronicle alias add team catalog://127.0.0.1:8081 --ak USER_AK --sk USER_SK pchronicle alias set-url prod s3://new-bucket/evals @@ -114,8 +114,10 @@ For `http://` endpoints, pChronicle also enables `AWS_ALLOW_HTTP` automatically for local S3-compatible services such as MinIO. `alias set-url` accepts the same `--endpoint` option and preserves the existing endpoint when changing between two S3 URIs without specifying a new one. -The optional `--region` is also stored per alias; when omitted, the S3 client -uses its default region (`us-west-2` when a fallback is required). +The optional `--region` is stored per alias. When an `s3://` alias omits it, +pChronicle applies `us-west-2` as `AWS_REGION` / `AWS_DEFAULT_REGION` before +opening the store. Endpoint, region, and credentials are applied before the +Tokio runtime starts so OpenDAL sees them reliably. ### Inspect and find @@ -293,6 +295,9 @@ pchronicle serve [--gateway-stream-markdown] [--gateway-debug] [--catalog-config FILE] [<[NAME=]DATASET> ...] +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... +pchronicle serve catalog dataset list --catalog-config FILE pchronicle serve catalog issue --catalog-config FILE NAME pchronicle serve catalog grant --catalog-config FILE NAME DATASET... pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... @@ -309,13 +314,17 @@ pchronicle serve \ Every listener must use a loopback address. A bare single Dataset is mounted as `default`; with several Datasets, use `NAME=DATASET` when a stable mount name is needed. Control requires a mount named `default`. -`--catalog-config FILE` serves a path Directory instead of opening Datasets -in the parent process. Pair it with `alias add NAME catalog://127.0.0.1:PORT --ak --sk`. -`pchronicle serve catalog issue|grant|revoke` rewrites that file and does not -start HTTP; `issue` prints the user secret once. Restart serve after changing -users or grants. `catalog` is a reserved `serve` subcommand; mount a path of -that name as `./catalog`. -See [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md). +`--catalog-config FILE` mounts every `[datasets.*]` library in the Directory +file into Warehouse and enables `catalog://` locators. It conflicts with +positional Dataset mounts. Pair Directory clients with +`alias add NAME catalog://127.0.0.1:PORT --ak --sk`. +`pchronicle serve catalog dataset add|remove|list` and +`issue|grant|revoke` rewrite that file and do not start HTTP; `issue` prints +the user secret once. Restart serve after changing libraries, users, or grants. +`catalog` is a reserved `serve` subcommand; mount a path of that name as +`./catalog`. +See [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md) and +[RFC-0015](../../rfcs/0015-chronicle-manifest.md) for nested discovery sidecars. The config-free Gateway accepts canonical trajectory events at `POST /v1/events`. `--gateway-dataset` is an output URI and is auto-mounted; it is no longer a mounted Dataset name. Split templates accept the exact @@ -340,19 +349,20 @@ construction is explained in [Snapshot design](../design/catalog.md). #### Catalog management -Catalog configuration contains only users, Datasets, and grants. Management commands create the file when it does not exist. +The Directory ACL file contains users, datasets (libraries), and grants. +Management commands create the file when it does not exist. ```text -pchronicle serve catalog user create --catalog-config FILE NAME -pchronicle serve catalog user list --catalog-config FILE -pchronicle serve catalog user remove --catalog-config FILE NAME -pchronicle serve catalog dataset create --catalog-config FILE NAME URI [OPTIONS] +pchronicle serve catalog issue --catalog-config FILE NAME +pchronicle serve catalog grant --catalog-config FILE NAME DATASET... +pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI + [--endpoint URL] [--region REGION] [--access-key KEY] [--secret-key KEY] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog dataset show --catalog-config FILE NAME -pchronicle serve catalog dataset remove --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE USER DATASET --permission PERMISSION... -pchronicle serve catalog revoke --catalog-config FILE USER DATASET --permission PERMISSION... -pchronicle serve catalog grants --catalog-config FILE ``` -`user create` generates AK/SK and prints the secret once. `dataset create` registers the URI and storage credentials without creating or deleting backend data. `grant` and `revoke` manage `read`, `query`, `analyze`, `write`, and `admin` permissions. +`issue` generates a user AK/SK and prints the secret once. `dataset add` +registers the URI and optional backend storage credentials without creating or +deleting object-store data. `grant` / `revoke` add or remove library names on +that user (v1 grants are library membership, not fine-grained permission flags). diff --git a/docs/src/zh/pchronicle/design/architecture.md b/docs/src/zh/pchronicle/design/architecture.md index 3c9ec6b6..30758699 100644 --- a/docs/src/zh/pchronicle/design/architecture.md +++ b/docs/src/zh/pchronicle/design/architecture.md @@ -130,9 +130,11 @@ Server 静态挂载命名 path。Refresh 先完整构造新 Snapshot,再切换 Dataset table 先按 Source 裁剪,再打开命中的固定 version;cache 和 routing index 与 Snapshot generation 绑定。 -使用 `--catalog-config` 时,父进程只提供 Directory 列表/换票,自己不打开这些 path。 -已授权的 Web 查询在只含该用户 path 的 worker 中执行。CLI 换票后打开票里的 `uri`(一条 path) -并注入存储钥。这是 path 上的平台寻址,不是新的 Dataset 种类。 +使用 `--catalog-config` 时,Warehouse 会把 Directory ACL 文件中的全部 +`[datasets.*]` library 挂进数据面(与位置参数挂载等价),并同时提供 +`catalog://` 列表/换票路由。文件中的 S3 endpoint、region 与后端密钥在打开存储前 +写入进程环境。CLI 换票后打开票里的 `uri`(一条 path)并注入存储钥。这是 path 上的 +平台寻址,不是新的 Dataset 种类。 Web 与 API 是同一读取模型的 consumer,不形成新事实源。未知 API route 保持 error,不进入 SPA fallback;只接受 loopback listener。 @@ -154,7 +156,8 @@ SPA fallback;只接受 loopback listener。 ## 相关设计 - [Snapshot 设计](catalog.md):discovery、Snapshot 构造、惰性 Source resolve 与裁剪。 -- [RFC-0013 path Directory](../../rfcs/0013-pchronicle-warehouse-catalog.md):名字→path、ACL、换票与 query worker。 +- [RFC-0013 path Directory](../../rfcs/0013-pchronicle-warehouse-catalog.md):名字→path、ACL、换票。 +- [RFC-0015 `chronicle.manifest`](../../rfcs/0015-chronicle-manifest.md):嵌套 Dataset 发现与聚合统计 sidecar。 - [运行存储](trajectory-storage.md):canonical fact、存储布局与写入 ownership。 - [Storyline Lance](storyline-lance.md):三表 projection、内容层、发布与维护。 - [记录数据、视图与版本](../concepts/facts-and-projections.md):这些层次的用户心智模型。 diff --git a/docs/src/zh/pchronicle/design/catalog.md b/docs/src/zh/pchronicle/design/catalog.md index 7082f6df..47b07674 100644 --- a/docs/src/zh/pchronicle/design/catalog.md +++ b/docs/src/zh/pchronicle/design/catalog.md @@ -194,6 +194,12 @@ Catalog 产生四个 source: `live` 和 `events.lance` 的内部文件不会再次成为 source。这一“识别复合根后停止下探”的规则 避免把 manifest、generation、segment 或 `objects.lance` 错当成用户输入。 +当目录含有 `chronicle.manifest` +([RFC-0015](../../rfcs/0015-chronicle-manifest.md))时,discovery 优先采用该 sidecar: +`leaf` 且 `format = compact-jsonl/v1` 时可不打开 Lance 即归类为 Compact source; +`branch` 只扫描同样含有 sidecar 的一层子目录。Explorer 目录合计可对 leaf +`record_count` 做读侧汇总;写入方只更新 leaf manifest,不回写祖先。 + ### 5.2 本地发现 本地 URI 支持普通路径、`local://` 和 `file://`: diff --git a/docs/src/zh/pchronicle/guides/exchange.md b/docs/src/zh/pchronicle/guides/exchange.md index 66069fbe..e3e88e9e 100644 --- a/docs/src/zh/pchronicle/guides/exchange.md +++ b/docs/src/zh/pchronicle/guides/exchange.md @@ -7,7 +7,10 @@ export 拒绝这两种格式。 Compact JSONL 每行保留一个 JSON object,不赋予轨迹语义。使用 `--input-format compact-jsonl` 或 `--output-format compact-jsonl`;`--column` 映射与 snapshot sync -限制见[命令参考](../reference/cli.md)。 +限制见[命令参考](../reference/cli.md)。缺少可用 `id` 的记录会生成稳定的 +`source_filename#line_number`;export 按原始输入字节保留记录。compact import 成功后还会在 +dataset 根写入 leaf `chronicle.manifest`,便于后续 discovery 不必仅为分类打开 Lance +([RFC-0015](../../rfcs/0015-chronicle-manifest.md))。 ## 导入到新 Dataset diff --git a/docs/src/zh/pchronicle/guides/serve.md b/docs/src/zh/pchronicle/guides/serve.md index 82b230f8..1f7a4cc6 100644 --- a/docs/src/zh/pchronicle/guides/serve.md +++ b/docs/src/zh/pchronicle/guides/serve.md @@ -14,6 +14,9 @@ pchronicle serve [--gateway-stream-markdown] [--gateway-debug] [--catalog-config FILE] [<[NAME=]DATASET> ...] +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... +pchronicle serve catalog dataset list --catalog-config FILE pchronicle serve catalog issue --catalog-config FILE NAME pchronicle serve catalog grant --catalog-config FILE NAME DATASET... pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... @@ -45,25 +48,38 @@ Mount name 会成为 SQL schema 和 API 名称。需要稳定名称时使用 `NA ## 启动 Directory ```bash +pchronicle serve catalog dataset add \ + --catalog-config catalog.toml prod \ + --uri s3://bucket/prod \ + --endpoint http://127.0.0.1:9000 \ + --region us-west-2 \ + --access-key BACKEND_AK \ + --secret-key BACKEND_SK pchronicle serve catalog issue --catalog-config catalog.toml alice pchronicle serve catalog grant --catalog-config catalog.toml alice prod evals pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 ``` -`catalog.toml` 列出 libraries(每条都是 path)和 users。`serve catalog issue` 写入一个无授权 -用户,并把 sk 只打印到这次 stdout;`grant` / `revoke` 改 `datasets`,不启动 HTTP。改文件后 -必须重启 serve。父进程不打开这些 path。Web UI 通过请求头发送用户 ak/sk;查询在一次性 -worker 中执行,worker 只拿到该用户被授权的 path。另一终端: +`catalog.toml` 列出 libraries(`[datasets.*]`,本地 path 或 `s3://`)和 users。 +`serve catalog dataset add|remove|list` 改写 libraries,不启动 HTTP。 +`serve catalog issue` 写入一个无授权用户,并把 sk 只打印到这次 stdout; +`grant` / `revoke` 改该用户可打开的 library 名称。改文件后必须重启 serve。 + +`pchronicle serve --catalog-config` 会把文件中的 **全部** library 挂进 Warehouse +(与位置参数挂载等价),并启用 `catalog://` 换票路由。不要与位置参数 Dataset +同时使用。文件里的 S3 endpoint / region / 后端密钥会在打开存储前写入进程环境。 +使用 Directory 用户钥时,Web UI 可通过请求头发送 ak/sk。另一终端: ```bash pchronicle alias add team catalog://127.0.0.1:8081 --ak USER_AK --sk USER_SK pchronicle query @team/prod --sql 'SELECT 1' ``` -`@team` 是 Directory locator,不是 Dataset。`@team/prod` 换票后打开票里的 `uri`(一条 path)。 -同一 Directory 文件里所有 `s3://` 库必须共用同一组 endpoint、region 和后端密钥。 -listener 仍只允许 loopback。设计见 -[RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。 +`@team` 是 Directory locator,不是 Dataset。`@team/prod` 换票后打开票里的 `uri` +(一条 path)。同一 Directory 文件里所有 `s3://` 库必须共用同一组 endpoint、region +和后端密钥。listener 仍只允许 loopback。嵌套 Dataset 发现可使用 +`chronicle.manifest`([RFC-0015](../../rfcs/0015-chronicle-manifest.md))。 +Directory 设计见 [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。 ## 启用 Control 或 Gateway 集成 diff --git a/docs/src/zh/pchronicle/guides/ui.md b/docs/src/zh/pchronicle/guides/ui.md index 215f4c25..26df8591 100644 --- a/docs/src/zh/pchronicle/guides/ui.md +++ b/docs/src/zh/pchronicle/guides/ui.md @@ -123,10 +123,10 @@ Storage 是高级诊断页,不是日常浏览 Run 的必经步骤。左侧按 清除该站点的浏览器数据也会清除这份设置。Assistant 标记为 **Read-only · selected run data**, 用于解释当前上下文,不会改写 Dataset。 -使用 `pchronicle serve --catalog-config` 时,从左侧 **Keys** 打开设置,填写 Directory 用户的 -access key 和 secret key。它们保存在 `localStorage`,并作为 -`x-pchronicle-access-key` / `x-pchronicle-secret-key` 发给当前 pChronicle 服务端。 -它们决定浏览器可以打开哪些 path,不是对象存储后端密钥。 +使用 `pchronicle serve --catalog-config` 时,ACL 文件中的全部 library 已挂载供本机浏览。 +若需要 Directory 用户鉴权流程,从左侧 **Keys** 填写 access key 和 secret key。它们保存在 +`localStorage`,并作为 `x-pchronicle-access-key` / `x-pchronicle-secret-key` 发给当前 +pChronicle 服务端。它们决定浏览器可打开哪些 Directory path,不是对象存储后端密钥。 ## 常见问题 diff --git a/docs/src/zh/pchronicle/reference/cases-platform.md b/docs/src/zh/pchronicle/reference/cases-platform.md index 93c99960..9b8d228f 100644 --- a/docs/src/zh/pchronicle/reference/cases-platform.md +++ b/docs/src/zh/pchronicle/reference/cases-platform.md @@ -1,44 +1,45 @@ -# pChronicle 集群平台与 Catalog Server 场景 +# pChronicle Directory 与平台场景 -本文覆盖平台化部署。Catalog 配置只管理用户、Dataset 和授权;Warehouse 的服务参数仍由 `pchronicle serve` 提供。 +面向平台部署的 Directory 配置。ACL 文件管理用户、datasets(libraries)和授权; +Warehouse 的 listen / Gateway 参数仍由 `pchronicle serve` 提供。 -## P01:从空配置创建 Catalog 用户 +## P01:从空配置签发 Directory 用户 ```bash -pchronicle serve catalog user create \ +pchronicle serve catalog issue \ --catalog-config ./catalog.toml alice ``` -如果文件不存在,命令创建配置文件、生成用户 AK/SK,并只在本次输出 secret。 +如果文件不存在,命令会创建配置文件、写入无授权用户,并只在本次 stdout 打印 secret。 -## P02:登记 Dataset +## P02:登记 Dataset library ```bash -pchronicle serve catalog dataset create \ +pchronicle serve catalog dataset add \ --catalog-config ./catalog.toml \ - prod s3://bucket/prod \ + prod \ + --uri s3://bucket/prod \ --endpoint http://127.0.0.1:9000 \ --region us-west-2 \ - --ak BACKEND_AK \ - --sk BACKEND_SK + --access-key BACKEND_AK \ + --secret-key BACKEND_SK ``` -该命令只登记 Dataset,不创建或删除后端数据。 +该命令只登记 URI 与后端凭据,不创建或删除对象存储数据。同一文件中所有 `s3://` +library 必须共用同一组 endpoint、region 和后端密钥。 -## P03:授权用户 +## P03:给用户授权 library ```bash pchronicle serve catalog grant \ --catalog-config ./catalog.toml \ - alice prod \ - --permission read \ - --permission query \ - --permission analyze + alice prod ``` -预期:配置中出现独立的 `[[grants]]` 记录。 +预期:出现 `[[grants]]`,该用户可打开 `prod`。v1 授权是库成员关系,不是 +`--permission` 细粒度标志。 -## P04:启动 Catalog Server +## P04:用 catalog 挂载启动 serve ```bash pchronicle serve \ @@ -46,9 +47,10 @@ pchronicle serve \ --listen 127.0.0.1:8081 ``` -父进程负责用户认证、Dataset 列表和 ticket;查询数据面在授权 mounts 的 worker 中执行。 +文件中每个 `[datasets.*]` 都会挂进 Warehouse;`catalog://` 换票路由仍可用。 +改 ACL 后需重启 serve。 -## P05:访问授权 Dataset +## P05:经 Directory alias 访问授权 Dataset ```bash pchronicle alias add team catalog://127.0.0.1:8081 \ @@ -57,17 +59,17 @@ pchronicle query @team/prod \ --sql 'SELECT COUNT(*) AS runs FROM dataset.runs' ``` -预期:授权用户可以查询 `prod`;未授权用户或未知 Dataset 返回相同的 404 资源错误。 +预期:授权用户可查询 `prod`;未知 Dataset 失败关闭。 -## P06:撤销授权 +## P06:撤销 library 授权 ```bash pchronicle serve catalog revoke \ --catalog-config ./catalog.toml \ - alice prod --permission query + alice prod ``` -预期:后续查询被拒绝,但 `read` 和其它仍保留的权限不受影响。 +预期:该用户后续无法再打开 `@team/prod`。 ## P07:RustFS Warehouse 回归 @@ -80,13 +82,14 @@ export PCHRONICLE_RUSTFS_SECRET_KEY=rustfsadmin export PCHRONICLE_RUSTFS_BUCKET=pchronicle-cases ``` -然后运行 RustFS 回归测试,验证 Dataset 写入、Catalog discovery、SQL 查询、Explorer 和 refresh 行为。 +然后跑 RustFS 回归,覆盖 Dataset 写入、Snapshot discovery(含 +`chronicle.manifest`)、SQL、Explorer 与 refresh。 平台验收重点: -- Catalog 文件可从空文件开始构建; -- 用户、Dataset 和 grants 修改是确定性的; -- Dataset 后端凭据只在授权 ticket 中使用; -- Worker 只收到当前用户被授权的 mounts; -- Catalog refresh 不影响已完成查询的 snapshot; -- RustFS 上的 Warehouse 行为与本地 Dataset 一致。 +- ACL 可从空文件开始构建; +- 用户、dataset 与 grants 修改是确定性的; +- 后端对象存储密钥留在 catalog 文件 / ticket 路径,不出现在 `alias list`; +- `--catalog-config` serve 会挂载全部已登记 library; +- Snapshot refresh 不改动进行中查询的 Snapshot; +- 覆盖路径上 RustFS Warehouse 行为与本地 Dataset 一致。 diff --git a/docs/src/zh/pchronicle/reference/cli.md b/docs/src/zh/pchronicle/reference/cli.md index daacab2c..e7021404 100644 --- a/docs/src/zh/pchronicle/reference/cli.md +++ b/docs/src/zh/pchronicle/reference/cli.md @@ -139,7 +139,7 @@ pchronicle alias [list|add|remove|rename|get-url|set-url] [ARGUMENTS] pchronicle alias add local ./trajectory-data pchronicle alias add prod s3://bucket/evals pchronicle alias add secure s3://bucket/evals --ak "$AWS_ACCESS_KEY_ID" --sk "$AWS_SECRET_ACCESS_KEY" -pchronicle alias add minio s3://bucket/evals --endpoint http://127.0.0.1:9000 --ak 123 --sk 123 +pchronicle alias add minio s3://bucket/evals --endpoint http://127.0.0.1:9000 --region us-west-2 --ak 123 --sk 123 pchronicle alias add regional s3://bucket/evals --region us-west-2 pchronicle alias add team catalog://127.0.0.1:8081 --ak USER_AK --sk USER_SK pchronicle alias @@ -167,7 +167,9 @@ Agent 会话目录。 [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。 `alias set-url` 也支持相同的 `--endpoint` 参数;在两个 S3 URI 之间切换且未指定新 endpoint 时, 会保留原有 endpoint。 -可选的 `--region` 也会按 alias 保存;省略时由 S3 客户端自行处理,需要回退时默认使用 `us-west-2`。 +可选的 `--region` 按 alias 保存;`s3://` alias 省略时,打开存储前会把 +`AWS_REGION` / `AWS_DEFAULT_REGION` 设为 `us-west-2`。endpoint、region 与凭证在 Tokio +runtime 启动前写入进程环境,避免 OpenDAL 读不到 region。 ### 2.4 `ls` @@ -416,6 +418,9 @@ pchronicle serve [--gateway-stream-markdown] [--gateway-debug] [--catalog-config FILE] [<[NAME=]DATASET> ...] +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... +pchronicle serve catalog dataset list --catalog-config FILE pchronicle serve catalog issue --catalog-config FILE NAME pchronicle serve catalog grant --catalog-config FILE NAME DATASET... pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... @@ -431,12 +436,13 @@ pchronicle serve \ 未指定服务 flag 时,只读 Web/API 默认监听 `127.0.0.1:0`。多个 Dataset 使用 `NAME=DATASET` mount;Control 模式要求名为 `default` 的 mount。`--catalog-config FILE` -以 Directory 方式服务,父进程不打开 Datasets;配合 -`alias add NAME catalog://127.0.0.1:PORT --ak --sk`。 -`pchronicle serve catalog issue|grant|revoke` 只改该文件、不启动 HTTP;`issue` 把用户 -sk 只打印一次。改用户或授权后必须重启 serve。`catalog` 是 `serve` 的保留子命令,挂载同名 -路径请用 `./catalog`。见 -[RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。无需配置的 `--gateway` +会把文件中全部 `[datasets.*]` 挂进 Warehouse,并启用 `catalog://` locator;不能与位置参数 +Dataset 同时使用。配合 `alias add NAME catalog://127.0.0.1:PORT --ak --sk`。 +`pchronicle serve catalog dataset add|remove|list` 与 `issue|grant|revoke` 只改该文件、 +不启动 HTTP;`issue` 把用户 sk 只打印一次。改 library、用户或授权后必须重启 serve。 +`catalog` 是 `serve` 的保留子命令,挂载同名路径请用 `./catalog`。见 +[RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md) 与 +[RFC-0015](../../rfcs/0015-chronicle-manifest.md)。无需配置的 `--gateway` 在 `POST /v1/events` 接收 canonical trajectory events;`--gateway-dataset` 是自动挂载的 输出 URI,不再是 mount name。`--gateway-split` 支持 `{user}`、`{date}`、`{hour}`。 已有 canonical source 默认在最后一条事件后空闲 30 分钟才自动刷新 Storyline projection; @@ -449,22 +455,21 @@ loopback;服务准备完成后,stdout 输出一行版本化 readiness JSON #### Catalog 管理 -Catalog 配置只包含用户、Dataset 和授权关系。配置文件不存在时,管理命令会自动创建。 +Directory ACL 文件包含用户、datasets(libraries)和 grants。配置文件不存在时,管理命令会自动创建。 ```text -pchronicle serve catalog user create --catalog-config FILE NAME -pchronicle serve catalog user list --catalog-config FILE -pchronicle serve catalog user remove --catalog-config FILE NAME -pchronicle serve catalog dataset create --catalog-config FILE NAME URI [OPTIONS] +pchronicle serve catalog issue --catalog-config FILE NAME +pchronicle serve catalog grant --catalog-config FILE NAME DATASET... +pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI + [--endpoint URL] [--region REGION] [--access-key KEY] [--secret-key KEY] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog dataset show --catalog-config FILE NAME -pchronicle serve catalog dataset remove --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE USER DATASET --permission PERMISSION... -pchronicle serve catalog revoke --catalog-config FILE USER DATASET --permission PERMISSION... -pchronicle serve catalog grants --catalog-config FILE ``` -`user create` 生成 AK/SK 并只显示一次 secret;`dataset create` 只登记 URI 和存储凭据,不删除或创建后端数据;`grant`/`revoke` 管理 `read`、`query`、`analyze`、`write`、`admin` 权限。 +`issue` 生成用户 AK/SK 并只显示一次 secret;`dataset add` 只登记 URI 与可选后端存储凭据, +不创建或删除对象存储数据;`grant`/`revoke` 增减该用户可打开的 library 名称(v1 是库成员关系, +不是细粒度 `--permission` 标志)。 ### 公共输出与退出状态