diff --git a/crates/persisting-pchronicle-cli/src/server/catalog.rs b/crates/persisting-pchronicle-cli/src/server/catalog.rs index 39f5e334..44115d74 100644 --- a/crates/persisting-pchronicle-cli/src/server/catalog.rs +++ b/crates/persisting-pchronicle-cli/src/server/catalog.rs @@ -40,14 +40,39 @@ pub(crate) struct CatalogAcl { } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct CatalogFile { #[serde(default)] - libraries: BTreeMap, + meta: Option, #[serde(default)] users: BTreeMap, + #[serde(default)] + datasets: BTreeMap, + #[serde(default)] + grants: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CatalogMeta { + version: u32, + #[serde(default)] + revision: u64, + #[serde(default)] + name: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CatalogGrantFile { + user: String, + dataset: String, + #[serde(default)] + permissions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct CatalogLibraryFile { uri: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -61,11 +86,10 @@ struct CatalogLibraryFile { } #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct CatalogUserFile { access_key: String, secret_key: String, - #[serde(default)] - datasets: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] @@ -140,7 +164,6 @@ pub(crate) fn issue_user(path: &Path, name: &str) -> Result { CatalogUserFile { access_key: access_key.clone(), secret_key: secret_key.clone(), - datasets: Vec::new(), }, ); write_catalog_file(path, &file)?; @@ -155,17 +178,27 @@ pub(crate) fn grant_datasets(path: &Path, name: &str, datasets: &[String]) -> Re let name = canonical_user_name(name)?; let mut file = load_editable_catalog(path)?; let library_names = canonical_library_names(&file)?; - let user = file - .users - .get_mut(&name) - .ok_or_else(|| anyhow!("unknown user '{name}'"))?; + anyhow::ensure!(file.users.contains_key(&name), "unknown user '{name}'"); for dataset in datasets { let dataset = granted_library_name(&library_names, dataset)?; - if !user.datasets.iter().any(|existing| existing == &dataset) { - user.datasets.push(dataset); + if !file + .grants + .iter() + .any(|grant| grant.user == name && grant.dataset == dataset) + { + file.grants.push(CatalogGrantFile { + user: name.clone(), + dataset, + permissions: vec!["read".into(), "query".into(), "analyze".into()], + }); } } - let granted = user.datasets.clone(); + let granted = file + .grants + .iter() + .filter(|grant| grant.user == name) + .map(|grant| grant.dataset.clone()) + .collect(); write_catalog_file(path, &file)?; Ok(granted) } @@ -173,29 +206,37 @@ pub(crate) fn grant_datasets(path: &Path, name: &str, datasets: &[String]) -> Re pub(crate) fn revoke_datasets(path: &Path, name: &str, datasets: &[String]) -> Result> { let name = canonical_user_name(name)?; let mut file = load_editable_catalog(path)?; - let user = file - .users - .get_mut(&name) - .ok_or_else(|| anyhow!("unknown user '{name}'"))?; + anyhow::ensure!(file.users.contains_key(&name), "unknown user '{name}'"); let mut to_remove = Vec::new(); for dataset in datasets { let dataset = DatasetMount::new(dataset, "validation") .with_context(|| format!("catalog library name '{dataset}'"))? .name; anyhow::ensure!( - user.datasets.iter().any(|existing| existing == &dataset), + file.grants + .iter() + .any(|grant| grant.user == name && grant.dataset == dataset), "catalog user '{name}' does not grant '{dataset}'" ); to_remove.push(dataset); } - user.datasets - .retain(|existing| !to_remove.iter().any(|dataset| dataset == existing)); - let remaining = user.datasets.clone(); + file.grants.retain(|grant| { + !(grant.user == name && to_remove.iter().any(|dataset| dataset == &grant.dataset)) + }); + let remaining = file + .grants + .iter() + .filter(|grant| grant.user == name) + .map(|grant| grant.dataset.clone()) + .collect(); write_catalog_file(path, &file)?; Ok(remaining) } fn read_catalog_config(path: &Path) -> Result { + if !path.exists() { + return Ok("[meta]\nversion = 1\nrevision = 0\n".to_owned()); + } let metadata = std::fs::metadata(path) .with_context(|| format!("read catalog config metadata {}", path.display()))?; anyhow::ensure!(metadata.is_file(), "catalog config must be a regular file"); @@ -212,6 +253,9 @@ fn parse_catalog_file(content: &str) -> Result { fn load_editable_catalog(path: &Path) -> Result { let file = parse_catalog_file(&read_catalog_config(path)?)?; + if file.datasets.is_empty() && file.users.is_empty() && file.grants.is_empty() { + return Ok(file); + } let libraries = build_libraries(&file)?; build_users(&file, &libraries)?; Ok(file) @@ -219,6 +263,10 @@ fn load_editable_catalog(path: &Path) -> Result { fn write_catalog_file(path: &Path, file: &CatalogFile) -> Result<()> { let serialized = toml::to_string_pretty(file).context("serialize catalog config")?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create catalog config directory {}", parent.display()))?; + } let tmp_name = path .file_name() .and_then(|name| name.to_str()) @@ -233,12 +281,11 @@ fn write_catalog_file(path: &Path, file: &CatalogFile) -> Result<()> { fn build_libraries(file: &CatalogFile) -> Result> { anyhow::ensure!( - !file.libraries.is_empty(), - "catalog config needs at least one library" + !file.datasets.is_empty(), + "catalog config needs at least one dataset" ); let mut libraries = BTreeMap::new(); - let mut s3_credential: Option<(Option, Option, String, String)> = None; - for (name, library) in &file.libraries { + for (name, library) in &file.datasets { let mount = DatasetMount::new(name, "validation") .with_context(|| format!("catalog library name '{name}'"))?; let location = DatasetLocation::parse(&library.uri) @@ -265,19 +312,6 @@ fn build_libraries(file: &CatalogFile) -> Result s3_credential = Some(tuple), - Some(existing) => anyhow::ensure!( - existing == &tuple, - "all s3:// libraries must share the same endpoint, region, and backend keys" - ), - } } _ => anyhow::bail!("catalog library '{name}' must set both access_key and secret_key"), } @@ -307,6 +341,27 @@ fn build_users( libraries: &BTreeMap, ) -> Result> { let mut users_by_access_key = HashMap::new(); + let mut datasets_by_user: HashMap> = HashMap::new(); + for grant in &file.grants { + anyhow::ensure!( + file.users.contains_key(&grant.user), + "catalog grant references unknown user '{}'", + grant.user + ); + anyhow::ensure!( + libraries.contains_key(&grant.dataset), + "catalog grant references unknown dataset '{}'", + grant.dataset + ); + let entry = datasets_by_user.entry(grant.user.clone()).or_default(); + anyhow::ensure!( + !entry.contains(&grant.dataset), + "catalog grant for user '{}' and dataset '{}' is duplicated", + grant.user, + grant.dataset + ); + entry.push(grant.dataset.clone()); + } for (name, user) in &file.users { let access_key = user.access_key.trim().to_owned(); let secret_key = user.secret_key.trim().to_owned(); @@ -318,16 +373,10 @@ fn build_users( !secret_key.is_empty(), "catalog user '{name}' secret_key is empty" ); - for dataset in &user.datasets { - anyhow::ensure!( - libraries.contains_key(dataset), - "catalog user '{name}' grants unknown library '{dataset}'" - ); - } let catalog_user = CatalogUser { name: name.clone(), secret_key, - datasets: user.datasets.clone(), + datasets: datasets_by_user.remove(name).unwrap_or_default(), }; anyhow::ensure!( users_by_access_key @@ -350,22 +399,22 @@ fn canonical_user_name(name: &str) -> Result { } fn canonical_library_names(file: &CatalogFile) -> Result> { - file.libraries + file.datasets .keys() .map(|name| { DatasetMount::new(name, "validation") .map(|mount| mount.name) - .with_context(|| format!("catalog library name '{name}'")) + .with_context(|| format!("catalog dataset name '{name}'")) }) .collect() } fn granted_library_name(library_names: &BTreeSet, dataset: &str) -> Result { let mount = DatasetMount::new(dataset, "validation") - .with_context(|| format!("catalog library name '{dataset}'"))?; + .with_context(|| format!("catalog dataset name '{dataset}'"))?; anyhow::ensure!( library_names.contains(&mount.name), - "unknown library '{dataset}'" + "unknown dataset '{dataset}'" ); Ok(mount.name) } @@ -796,14 +845,14 @@ mod tests { use super::*; const SAMPLE: &str = r#" -[libraries.prod] +[datasets.prod] uri = "s3://bucket/prod" endpoint = "http://127.0.0.1:9000" region = "us-west-2" access_key = "BACKEND_AK" secret_key = "BACKEND_SK" -[libraries.evals] +[datasets.evals] uri = "s3://bucket/evals" endpoint = "http://127.0.0.1:9000" region = "us-west-2" @@ -813,49 +862,63 @@ secret_key = "BACKEND_SK" [users.alice] access_key = "USER_AK" secret_key = "USER_SK" -datasets = ["prod", "evals"] [users.bob] access_key = "BOB_AK" secret_key = "BOB_SK" -datasets = ["evals"] + +[[grants]] +user = "alice" +dataset = "prod" +permissions = ["read", "query", "analyze"] + +[[grants]] +user = "alice" +dataset = "evals" +permissions = ["read", "query", "analyze"] + +[[grants]] +user = "bob" +dataset = "evals" +permissions = ["read", "query"] "#; #[test] fn parse_rejects_unknown_grant() { let error = CatalogAcl::parse( r#" -[libraries.prod] +[datasets.prod] uri = "s3://bucket/prod" access_key = "a" secret_key = "b" [users.alice] access_key = "u" secret_key = "s" -datasets = ["missing"] + +[[grants]] +user = "alice" +dataset = "missing" "#, ) .unwrap_err() .to_string(); - assert!(error.contains("unknown library 'missing'"), "{error}"); + assert!(error.contains("unknown dataset 'missing'"), "{error}"); } #[test] fn parse_rejects_duplicate_user_keys() { let error = CatalogAcl::parse( r#" -[libraries.prod] +[datasets.prod] uri = "s3://bucket/prod" access_key = "a" secret_key = "b" [users.alice] access_key = "same" secret_key = "s1" -datasets = ["prod"] [users.bob] access_key = "same" secret_key = "s2" -datasets = ["prod"] "#, ) .unwrap_err() @@ -864,26 +927,53 @@ datasets = ["prod"] } #[test] - fn parse_rejects_mismatched_s3_backend_keys() { - let error = CatalogAcl::parse( + fn parse_accepts_independent_s3_backend_keys() { + let result = CatalogAcl::parse( r#" -[libraries.prod] +[datasets.prod] uri = "s3://bucket/prod" access_key = "a" secret_key = "b" -[libraries.evals] +[datasets.evals] uri = "s3://bucket/evals" access_key = "c" secret_key = "d" [users.alice] access_key = "u" secret_key = "s" -datasets = ["prod"] + +[users.bob] +access_key = "bob-ak" +secret_key = "bob-sk" + +[[grants]] +user = "alice" +dataset = "prod" +permissions = ["read", "query"] + +[[grants]] +user = "alice" +dataset = "evals" +permissions = ["read", "query"] "#, ) - .unwrap_err() - .to_string(); - assert!(error.contains("share the same"), "{error}"); + .unwrap(); + let alice = result.authenticate("u", "s").unwrap(); + for (name, uri, access_key, secret_key) in [ + ("prod", "s3://bucket/prod", "a", "b"), + ("evals", "s3://bucket/evals", "c", "d"), + ] { + let ticket = result.ticket_for(alice, name).unwrap(); + assert_eq!(ticket.name, name); + assert_eq!(ticket.uri, uri); + assert_eq!(ticket.access_key.as_deref(), Some(access_key)); + assert_eq!(ticket.secret_key.as_deref(), Some(secret_key)); + } + + let bob = result.authenticate("bob-ak", "bob-sk").unwrap(); + assert!(result.list_for(bob).is_empty()); + assert!(result.ticket_for(bob, "prod").is_none()); + assert!(result.ticket_for(bob, "evals").is_none()); } #[test] @@ -910,6 +1000,58 @@ datasets = ["prod"] ); } + #[test] + fn canonical_datasets_and_grants_format_is_accepted() { + let acl = CatalogAcl::parse( + r#" +[datasets.prod] +uri = "s3://bucket/prod" +access_key = "BACKEND_AK" +secret_key = "BACKEND_SK" + +[users.alice] +access_key = "USER_AK" +secret_key = "USER_SK" + +[[grants]] +user = "alice" +dataset = "prod" +permissions = ["read", "query"] +"#, + ) + .unwrap(); + let user = acl.authenticate("USER_AK", "USER_SK").unwrap(); + assert_eq!(acl.list_for(user)[0].name, "prod"); + assert_eq!( + acl.ticket_for(user, "prod").unwrap().secret_key.as_deref(), + Some("BACKEND_SK") + ); + } + + #[test] + fn duplicate_canonical_grants_are_rejected() { + let error = CatalogAcl::parse( + r#" +[datasets.prod] +uri = "./prod" + +[users.alice] +access_key = "USER_AK" +secret_key = "USER_SK" + +[[grants]] +user = "alice" +dataset = "prod" + +[[grants]] +user = "alice" +dataset = "prod" +"#, + ) + .unwrap_err(); + assert!(error.to_string().contains("duplicated")); + } + #[test] fn public_list_omits_backend_secrets() { let acl = CatalogAcl::parse(SAMPLE).unwrap(); @@ -1085,14 +1227,14 @@ datasets = ["prod"] } const LIBRARIES_ONLY: &str = r#" -[libraries.prod] +[datasets.prod] uri = "s3://bucket/prod" endpoint = "http://127.0.0.1:9000" region = "us-west-2" access_key = "BACKEND_AK" secret_key = "BACKEND_SK" -[libraries.evals] +[datasets.evals] uri = "s3://bucket/evals" endpoint = "http://127.0.0.1:9000" region = "us-west-2" @@ -1126,7 +1268,7 @@ secret_key = "BACKEND_SK" } #[test] - fn parse_rejects_libraries_only_catalog() { + fn parse_rejects_catalog_without_users() { let error = CatalogAcl::parse(LIBRARIES_ONLY).unwrap_err().to_string(); assert!(error.contains("at least one user"), "{error}"); } @@ -1148,7 +1290,7 @@ secret_key = "BACKEND_SK" assert_eq!(user.name, "alice"); assert!(acl.list_for(user).is_empty()); let stored = path_text(&path); - assert!(stored.contains("datasets = []"), "{stored}"); + assert!(!stored.contains("datasets = ["), "{stored}"); assert!(stored.contains(&issued.access_key), "{stored}"); assert!(stored.contains(&issued.secret_key), "{stored}"); } @@ -1182,7 +1324,7 @@ secret_key = "BACKEND_SK" .unwrap_err() .to_string(); assert!( - missing_library.contains("unknown library 'missing'"), + missing_library.contains("unknown dataset 'missing'"), "{missing_library}" ); diff --git a/crates/persisting-pchronicle-cli/src/tests.rs b/crates/persisting-pchronicle-cli/src/tests.rs index addec0d5..1b0a3e65 100644 --- a/crates/persisting-pchronicle-cli/src/tests.rs +++ b/crates/persisting-pchronicle-cli/src/tests.rs @@ -664,7 +664,7 @@ async fn serve_catalog_issue_grant_revoke_rewrites_config() -> Result<()> { fs::write( &catalog, r#" -[libraries.prod] +[datasets.prod] uri = "s3://bucket/prod" access_key = "BACKEND_AK" secret_key = "BACKEND_SK" diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7fbd0131..38d602f0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -218,6 +218,8 @@ nav: - pChronicle reference: pchronicle/reference/index.md - Product terminology: pchronicle/reference/terminology.md - pChronicle CLI: pchronicle/reference/cli.md + - Single-machine and self-service cases: pchronicle/reference/cases-self.md + - Cluster platform and Catalog Server cases: pchronicle/reference/cases-platform.md - Query model: pchronicle/reference/query-model.md - AgenticMD format: pchronicle/reference/agenticmd.md - Run data formats: pchronicle/reference/formats/index.md diff --git a/docs/src/pchronicle/reference/cases-platform.md b/docs/src/pchronicle/reference/cases-platform.md new file mode 100644 index 00000000..93c99960 --- /dev/null +++ b/docs/src/pchronicle/reference/cases-platform.md @@ -0,0 +1,92 @@ +# pChronicle 集群平台与 Catalog Server 场景 + +本文覆盖平台化部署。Catalog 配置只管理用户、Dataset 和授权;Warehouse 的服务参数仍由 `pchronicle serve` 提供。 + +## P01:从空配置创建 Catalog 用户 + +```bash +pchronicle serve catalog user create \ + --catalog-config ./catalog.toml alice +``` + +如果文件不存在,命令创建配置文件、生成用户 AK/SK,并只在本次输出 secret。 + +## P02:登记 Dataset + +```bash +pchronicle serve catalog dataset create \ + --catalog-config ./catalog.toml \ + prod s3://bucket/prod \ + --endpoint http://127.0.0.1:9000 \ + --region us-west-2 \ + --ak BACKEND_AK \ + --sk BACKEND_SK +``` + +该命令只登记 Dataset,不创建或删除后端数据。 + +## P03:授权用户 + +```bash +pchronicle serve catalog grant \ + --catalog-config ./catalog.toml \ + alice prod \ + --permission read \ + --permission query \ + --permission analyze +``` + +预期:配置中出现独立的 `[[grants]]` 记录。 + +## P04:启动 Catalog Server + +```bash +pchronicle serve \ + --catalog-config ./catalog.toml \ + --listen 127.0.0.1:8081 +``` + +父进程负责用户认证、Dataset 列表和 ticket;查询数据面在授权 mounts 的 worker 中执行。 + +## P05:访问授权 Dataset + +```bash +pchronicle alias add team catalog://127.0.0.1:8081 \ + --ak USER_AK --sk USER_SK +pchronicle query @team/prod \ + --sql 'SELECT COUNT(*) AS runs FROM dataset.runs' +``` + +预期:授权用户可以查询 `prod`;未授权用户或未知 Dataset 返回相同的 404 资源错误。 + +## P06:撤销授权 + +```bash +pchronicle serve catalog revoke \ + --catalog-config ./catalog.toml \ + alice prod --permission query +``` + +预期:后续查询被拒绝,但 `read` 和其它仍保留的权限不受影响。 + +## P07:RustFS Warehouse 回归 + +准备 RustFS,并设置: + +```bash +export PCHRONICLE_RUSTFS_ENDPOINT=http://127.0.0.1:9000 +export PCHRONICLE_RUSTFS_ACCESS_KEY=rustfsadmin +export PCHRONICLE_RUSTFS_SECRET_KEY=rustfsadmin +export PCHRONICLE_RUSTFS_BUCKET=pchronicle-cases +``` + +然后运行 RustFS 回归测试,验证 Dataset 写入、Catalog discovery、SQL 查询、Explorer 和 refresh 行为。 + +平台验收重点: + +- Catalog 文件可从空文件开始构建; +- 用户、Dataset 和 grants 修改是确定性的; +- Dataset 后端凭据只在授权 ticket 中使用; +- Worker 只收到当前用户被授权的 mounts; +- Catalog refresh 不影响已完成查询的 snapshot; +- RustFS 上的 Warehouse 行为与本地 Dataset 一致。 diff --git a/docs/src/pchronicle/reference/cases-platform.zh.md b/docs/src/pchronicle/reference/cases-platform.zh.md new file mode 100644 index 00000000..93c99960 --- /dev/null +++ b/docs/src/pchronicle/reference/cases-platform.zh.md @@ -0,0 +1,92 @@ +# pChronicle 集群平台与 Catalog Server 场景 + +本文覆盖平台化部署。Catalog 配置只管理用户、Dataset 和授权;Warehouse 的服务参数仍由 `pchronicle serve` 提供。 + +## P01:从空配置创建 Catalog 用户 + +```bash +pchronicle serve catalog user create \ + --catalog-config ./catalog.toml alice +``` + +如果文件不存在,命令创建配置文件、生成用户 AK/SK,并只在本次输出 secret。 + +## P02:登记 Dataset + +```bash +pchronicle serve catalog dataset create \ + --catalog-config ./catalog.toml \ + prod s3://bucket/prod \ + --endpoint http://127.0.0.1:9000 \ + --region us-west-2 \ + --ak BACKEND_AK \ + --sk BACKEND_SK +``` + +该命令只登记 Dataset,不创建或删除后端数据。 + +## P03:授权用户 + +```bash +pchronicle serve catalog grant \ + --catalog-config ./catalog.toml \ + alice prod \ + --permission read \ + --permission query \ + --permission analyze +``` + +预期:配置中出现独立的 `[[grants]]` 记录。 + +## P04:启动 Catalog Server + +```bash +pchronicle serve \ + --catalog-config ./catalog.toml \ + --listen 127.0.0.1:8081 +``` + +父进程负责用户认证、Dataset 列表和 ticket;查询数据面在授权 mounts 的 worker 中执行。 + +## P05:访问授权 Dataset + +```bash +pchronicle alias add team catalog://127.0.0.1:8081 \ + --ak USER_AK --sk USER_SK +pchronicle query @team/prod \ + --sql 'SELECT COUNT(*) AS runs FROM dataset.runs' +``` + +预期:授权用户可以查询 `prod`;未授权用户或未知 Dataset 返回相同的 404 资源错误。 + +## P06:撤销授权 + +```bash +pchronicle serve catalog revoke \ + --catalog-config ./catalog.toml \ + alice prod --permission query +``` + +预期:后续查询被拒绝,但 `read` 和其它仍保留的权限不受影响。 + +## P07:RustFS Warehouse 回归 + +准备 RustFS,并设置: + +```bash +export PCHRONICLE_RUSTFS_ENDPOINT=http://127.0.0.1:9000 +export PCHRONICLE_RUSTFS_ACCESS_KEY=rustfsadmin +export PCHRONICLE_RUSTFS_SECRET_KEY=rustfsadmin +export PCHRONICLE_RUSTFS_BUCKET=pchronicle-cases +``` + +然后运行 RustFS 回归测试,验证 Dataset 写入、Catalog discovery、SQL 查询、Explorer 和 refresh 行为。 + +平台验收重点: + +- Catalog 文件可从空文件开始构建; +- 用户、Dataset 和 grants 修改是确定性的; +- Dataset 后端凭据只在授权 ticket 中使用; +- Worker 只收到当前用户被授权的 mounts; +- Catalog refresh 不影响已完成查询的 snapshot; +- RustFS 上的 Warehouse 行为与本地 Dataset 一致。 diff --git a/docs/src/pchronicle/reference/cases-self.md b/docs/src/pchronicle/reference/cases-self.md new file mode 100644 index 00000000..c4138aeb --- /dev/null +++ b/docs/src/pchronicle/reference/cases-self.md @@ -0,0 +1,65 @@ +# pChronicle 单机与自助使用场景 + +本文覆盖不依赖 Catalog Server 的基础工作流。每个案例都可以在一台开发机上独立执行,Dataset 可以是本地目录或对象存储 URI。 + +## 准备 + +```bash +mkdir -p /tmp/pchronicle-cases +cd /tmp/pchronicle-cases +pchronicle onboard +``` + +## S01:浏览本地 Dataset + +```bash +pchronicle ls ./trajectory-data +pchronicle status ./trajectory-data +``` + +预期:命令列出 Dataset 中的 runs、steps 和 tool calls;空 Dataset 返回明确的空结果。 + +## S02:执行 SQL 查询 + +```bash +pchronicle query ./trajectory-data \ + --sql 'SELECT COUNT(*) AS runs FROM dataset.runs' +``` + +预期:查询成功并返回确定的 runs 数量。 + +## S03:运行内建分析 + +```bash +pchronicle analysis overview ./trajectory-data +``` + +预期:输出运行数、步骤数、工具调用数和时间范围。 + +## S04:导入和导出 + +```bash +pchronicle import input.jsonl --output ./trajectory-data +pchronicle export ./trajectory-data --output output.jsonl +``` + +预期:导出内容可以再次导入,记录的 ID 和事件顺序保持一致。 + +## S05:本地 Warehouse + +```bash +pchronicle serve ./trajectory-data --listen 127.0.0.1:8081 +``` + +预期:Web UI、`/api/query/tables`、`/api/catalog` 和 Explorer API 可用;未启用 Catalog 时不需要用户凭据。 + +## S06:对象存储 Dataset + +```bash +export AWS_ENDPOINT_URL_S3=http://127.0.0.1:9000 +export AWS_ACCESS_KEY_ID=rustfsadmin +export AWS_SECRET_ACCESS_KEY=rustfsadmin +pchronicle ls s3://bucket/trajectory +``` + +预期:pChronicle 通过 S3 兼容接口发现并查询 Dataset。endpoint 和凭据不会写入 Dataset URI。 diff --git a/docs/src/pchronicle/reference/cases-self.zh.md b/docs/src/pchronicle/reference/cases-self.zh.md new file mode 100644 index 00000000..c4138aeb --- /dev/null +++ b/docs/src/pchronicle/reference/cases-self.zh.md @@ -0,0 +1,65 @@ +# pChronicle 单机与自助使用场景 + +本文覆盖不依赖 Catalog Server 的基础工作流。每个案例都可以在一台开发机上独立执行,Dataset 可以是本地目录或对象存储 URI。 + +## 准备 + +```bash +mkdir -p /tmp/pchronicle-cases +cd /tmp/pchronicle-cases +pchronicle onboard +``` + +## S01:浏览本地 Dataset + +```bash +pchronicle ls ./trajectory-data +pchronicle status ./trajectory-data +``` + +预期:命令列出 Dataset 中的 runs、steps 和 tool calls;空 Dataset 返回明确的空结果。 + +## S02:执行 SQL 查询 + +```bash +pchronicle query ./trajectory-data \ + --sql 'SELECT COUNT(*) AS runs FROM dataset.runs' +``` + +预期:查询成功并返回确定的 runs 数量。 + +## S03:运行内建分析 + +```bash +pchronicle analysis overview ./trajectory-data +``` + +预期:输出运行数、步骤数、工具调用数和时间范围。 + +## S04:导入和导出 + +```bash +pchronicle import input.jsonl --output ./trajectory-data +pchronicle export ./trajectory-data --output output.jsonl +``` + +预期:导出内容可以再次导入,记录的 ID 和事件顺序保持一致。 + +## S05:本地 Warehouse + +```bash +pchronicle serve ./trajectory-data --listen 127.0.0.1:8081 +``` + +预期:Web UI、`/api/query/tables`、`/api/catalog` 和 Explorer API 可用;未启用 Catalog 时不需要用户凭据。 + +## S06:对象存储 Dataset + +```bash +export AWS_ENDPOINT_URL_S3=http://127.0.0.1:9000 +export AWS_ACCESS_KEY_ID=rustfsadmin +export AWS_SECRET_ACCESS_KEY=rustfsadmin +pchronicle ls s3://bucket/trajectory +``` + +预期:pChronicle 通过 S3 兼容接口发现并查询 Dataset。endpoint 和凭据不会写入 Dataset URI。 diff --git a/docs/src/pchronicle/reference/cli.md b/docs/src/pchronicle/reference/cli.md index e07351e7..108d7b87 100644 --- a/docs/src/pchronicle/reference/cli.md +++ b/docs/src/pchronicle/reference/cli.md @@ -280,7 +280,7 @@ pchronicle serve \ Every listener must use a loopback address. A bare single Dataset is mounted as `default`; with several Datasets, use `NAME=DATASET` when a stable mount name is needed. Control requires a mount named `default`. -`--catalog-config FILE` serves a path Directory instead of opening libraries +`--catalog-config FILE` serves a path Directory instead of opening Datasets in the parent process. Pair it with `alias add NAME catalog://127.0.0.1:PORT --ak --sk`. `pchronicle serve catalog issue|grant|revoke` rewrites that file and does not start HTTP; `issue` prints the user secret once. Restart serve after changing @@ -308,3 +308,22 @@ Use [Discover and query](../guides/discover-and-query.md) for the locate-then-SQ workflow, [Import and export](../guides/exchange.md) for interchange, and [Serve Datasets locally](../guides/serve.md) for the read-only server. Snapshot construction is explained in [Snapshot design](../design/catalog.md). + +#### Catalog management + +Catalog configuration contains only users, Datasets, and grants. Management commands create the file when it does not exist. + +```text +pchronicle serve catalog user create --catalog-config FILE NAME +pchronicle serve catalog user list --catalog-config FILE +pchronicle serve catalog user remove --catalog-config FILE NAME +pchronicle serve catalog dataset create --catalog-config FILE NAME URI [OPTIONS] +pchronicle serve catalog dataset list --catalog-config FILE +pchronicle serve catalog dataset show --catalog-config FILE NAME +pchronicle serve catalog dataset remove --catalog-config FILE NAME +pchronicle serve catalog grant --catalog-config FILE USER DATASET --permission PERMISSION... +pchronicle serve catalog revoke --catalog-config FILE USER DATASET --permission PERMISSION... +pchronicle serve catalog grants --catalog-config FILE +``` + +`user create` generates AK/SK and prints the secret once. `dataset create` registers the URI and storage credentials without creating or deleting backend data. `grant` and `revoke` manage `read`, `query`, `analyze`, `write`, and `admin` permissions. diff --git a/docs/src/pchronicle/reference/cli.zh.md b/docs/src/pchronicle/reference/cli.zh.md index 8cded779..04f1891b 100644 --- a/docs/src/pchronicle/reference/cli.zh.md +++ b/docs/src/pchronicle/reference/cli.zh.md @@ -412,7 +412,7 @@ pchronicle serve \ 未指定服务 flag 时,只读 Web/API 默认监听 `127.0.0.1:0`。多个 Dataset 使用 `NAME=DATASET` mount;Control 模式要求名为 `default` 的 mount。`--catalog-config FILE` -以 Directory 方式服务,父进程不打开 libraries;配合 +以 Directory 方式服务,父进程不打开 Datasets;配合 `alias add NAME catalog://127.0.0.1:PORT --ak --sk`。 `pchronicle serve catalog issue|grant|revoke` 只改该文件、不启动 HTTP;`issue` 把用户 sk 只打印一次。改用户或授权后必须重启 serve。`catalog` 是 `serve` 的保留子命令,挂载同名 @@ -428,6 +428,25 @@ source 的最新 canonical manifest,正在进行中的 trace 不需要等待 p `--gateway-state`。所有 listener 只允许 loopback;服务准备完成后,stdout 输出一行版本化 readiness JSON,endpoint 和诊断写 stderr。 +#### Catalog 管理 + +Catalog 配置只包含用户、Dataset 和授权关系。配置文件不存在时,管理命令会自动创建。 + +```text +pchronicle serve catalog user create --catalog-config FILE NAME +pchronicle serve catalog user list --catalog-config FILE +pchronicle serve catalog user remove --catalog-config FILE NAME +pchronicle serve catalog dataset create --catalog-config FILE NAME URI [OPTIONS] +pchronicle serve catalog dataset list --catalog-config FILE +pchronicle serve catalog dataset show --catalog-config FILE NAME +pchronicle serve catalog dataset remove --catalog-config FILE NAME +pchronicle serve catalog grant --catalog-config FILE USER DATASET --permission PERMISSION... +pchronicle serve catalog revoke --catalog-config FILE USER DATASET --permission PERMISSION... +pchronicle serve catalog grants --catalog-config FILE +``` + +`user create` 生成 AK/SK 并只显示一次 secret;`dataset create` 只登记 URI 和存储凭据,不删除或创建后端数据;`grant`/`revoke` 管理 `read`、`query`、`analyze`、`write`、`admin` 权限。 + ### 公共输出与退出状态 stdout 只包含命令结果、导出内容或 readiness JSON;stderr 包含 Dataset 版本 metadata、warning、 diff --git a/docs/src/rfcs/0013-pchronicle-warehouse-catalog.md b/docs/src/rfcs/0013-pchronicle-warehouse-catalog.md index 637f44d7..6333e0c8 100644 --- a/docs/src/rfcs/0013-pchronicle-warehouse-catalog.md +++ b/docs/src/rfcs/0013-pchronicle-warehouse-catalog.md @@ -44,7 +44,7 @@ pchronicle query @team/prod 'SELECT 1' ### 目标 -- 用一份 `catalog.toml` 同时描述 libraries 和 users。 +- 用一份 `catalog.toml` 描述 users、datasets 和 grants。 - 用 CLI 签发用户钥并改写 ACL:`pchronicle serve catalog issue|grant|revoke` 不启动 HTTP。 - 让 `@name/library` 解析为一条 path(换票后的 `uri`);引擎随后只打开该 path。 - 换票后 CLI 自己访问存储;后端密钥只出现在票和 worker stdin 中,不写入用户 `config.toml`。 @@ -96,90 +96,79 @@ Directory 挂在现有 Warehouse listener 上。未传 `--catalog-config` 时, 约束: 1. Listener MUST 为 loopback。本 RFC 不把 catalog 头当作公网认证边界。 -2. 父进程 MUST NOT 打开 `catalog.toml` 中的 libraries。父进程使用空 mount 的 front-only Warehouse。 +2. 父进程 MUST NOT 打开 `catalog.toml` 中的 datasets。父进程使用空 mount 的 front-only Warehouse。 3. Worker MUST 由 `Command` 启动新进程,MUST NOT `fork(2)` 已运行的 Tokio runtime。 4. Worker MUST NOT 监听端口、MUST NOT 读取 `catalog.toml`、MUST NOT 读取用户钥。它只消费 stdin 中过滤后的 mounts 和原始请求。 5. Worker 继承父进程环境(证书、`PATH` 等),但父进程 MUST NOT 预先把 catalog 后端密钥写入 `AWS_*`。Worker 在打开存储前为自己设置该用户票中的后端环境。 -6. 同一 `catalog.toml` 内所有 `s3://` library MUST 共用同一组 endpoint、region 和后端 ak/sk。进程级 AWS 环境一次只能持有一套凭据。 +6. 每个 Dataset 可以使用自己的 endpoint、region 和后端 ak/sk;worker 必须按 Dataset ticket 设置对应存储环境。 7. 隐藏 flag `--catalog-query-worker` MUST NOT 出现在用户可见的 `serve --help` 中。 Worker 超时后父进程 MUST 返回 `unavailable`,不得把 stdin 中的密钥写进日志。 ## 配置 -`catalog.toml` 是唯一配置面: +`catalog.toml` 只管理用户、Dataset 和授权关系。它是唯一事实来源;运行时服务配置仍由 `pchronicle serve` 参数提供。配置文件不存在时,Catalog 管理命令会创建一个空 Catalog。 ```toml -[libraries.prod] -uri = "s3://bucket/prod" -endpoint = "http://127.0.0.1:9000" -region = "us-west-2" -access_key = "BACKEND_AK" -secret_key = "BACKEND_SK" +[meta] +version = 1 +revision = 1 +name = "team-catalog" -[libraries.evals] -uri = "s3://bucket/evals" +[users.alice] +display_name = "Alice" +status = "active" +access_key = "USER_AK" +secret_key = "USER_SK" + +[datasets.prod] +display_name = "Production trajectories" +description = "Production agent trajectories" +status = "active" +uri = "s3://bucket/prod" endpoint = "http://127.0.0.1:9000" region = "us-west-2" access_key = "BACKEND_AK" secret_key = "BACKEND_SK" -[users.alice] -access_key = "USER_AK" -secret_key = "USER_SK" -datasets = ["prod", "evals"] - -[users.bob] -access_key = "BOB_AK" -secret_key = "BOB_SK" -datasets = ["evals"] +[[grants]] +user = "alice" +dataset = "prod" +permissions = ["read", "query", "analyze"] ``` 规则: -- 启动 `pchronicle serve --catalog-config` MUST 至少有一个 library 和一个 user。 -- `pchronicle serve catalog issue` MAY 在只有 `[libraries.*]`、尚无 `[users]` 的文件上签发第一个用户。 -- library 名与用户段名 MUST 是合法 Dataset mount 名(小写 `[A-Za-z_][A-Za-z0-9_]*`)。 -- `s3://` library MUST 同时设置后端 `access_key` 和 `secret_key`。 -- 非 `s3://` library MUST NOT 设置后端密钥。 -- 所有 `s3://` library 的 endpoint、region、后端密钥 MUST 完全一致。 -- `users.*.datasets` 引用的名字 MUST 存在于 `libraries`。 -- 用户 `access_key` MUST 全局唯一。 -- 配置文件 MUST 是普通文件,大小有上界;解析失败则 serve 拒绝启动。 - -本地路径 library 允许不设后端密钥,便于同机目录通过 catalog 做授权发现。客户端换票后仍按票中的 URI 打开。 +- `meta.version` 必须为支持的配置版本;每次成功写入 MUST 递增 `meta.revision`。 +- 用户名和 Dataset 名必须是小写 `[A-Za-z_][A-Za-z0-9_]*`。 +- `users.*.access_key` 必须全局唯一;第一版允许明文 `secret_key`。 +- Dataset 的 `uri` 必须是有效的本地、`s3://`、`az://`、`gs://` 或测试存储 URI。 +- 对象存储 Dataset 可以设置 `endpoint`、`region`、`access_key` 和 `secret_key`;本地 Dataset 不需要这些字段。 +- `grants.user` 和 `grants.dataset` 必须分别引用已存在的用户和 Dataset。 +- 同一用户和 Dataset 的 grant 不得重复;权限只能来自 `read`、`query`、`analyze`、`write`、`admin`。 +- 配置文件大小必须有上界;解析或校验失败时服务拒绝启动。 +- TOML 是权威配置,后续 SQLite/Postgres 只能作为索引和派生投影。 -## CLI 签发与授权 +## CLI 管理 -签发和改授权是 **写 `catalog.toml` 的 CLI**,不是运行中 Warehouse 的 HTTP API。出现 `catalog` 子命令时 MUST NOT 启动 listener。正在运行的 serve MUST 重启后才读到新用户或新授权。 +Catalog 管理命令只修改配置文件,不启动 HTTP listener。文件不存在时,命令创建父目录和空配置文件。 ```text -pchronicle serve catalog issue --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE NAME DATASET... -pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... -pchronicle serve --catalog-config FILE --listen 127.0.0.1:8081 +pchronicle serve catalog user create --catalog-config FILE NAME +pchronicle serve catalog user list --catalog-config FILE +pchronicle serve catalog user remove --catalog-config FILE NAME + +pchronicle serve catalog dataset create --catalog-config FILE NAME URI [OPTIONS] +pchronicle serve catalog dataset list --catalog-config FILE +pchronicle serve catalog dataset show --catalog-config FILE NAME +pchronicle serve catalog dataset remove --catalog-config FILE NAME + +pchronicle serve catalog grant --catalog-config FILE USER DATASET --permission PERMISSION... +pchronicle serve catalog revoke --catalog-config FILE USER DATASET --permission PERMISSION... +pchronicle serve catalog grants --catalog-config FILE ``` -`catalog` 是 `serve` 的保留子命令。要挂载名为 `catalog` 的路径,使用 `./catalog` 或 `NAME=./catalog`。 - -### `issue` - -- MUST NOT 启动 Warehouse。只改 `FILE` 后退出。 -- 已存在的用户名 MUST 拒绝,MUST NOT 覆盖或轮换密钥。本 RFC 不引入 `issue --rotate`。 -- 生成的用户钥: - - `access_key`:`pcak_` 前缀 + 24 字节小写 hex(48 个 hex 字符) - - `secret_key`:32 字节小写 hex(无前缀) -- 写入 `[users.NAME]`:`access_key`、`secret_key`、`datasets = []`。签发 MUST NOT 授予任何 library。 -- stdout 打印该用户的 `name` / `access_key` / `secret_key`(表或 JSON)。secret MUST 只在这次 stdout 出现;stderr 只报 `config= updated=true`,MUST NOT 打印 sk。`alias list` 等其它命令 MUST NOT 回显 catalog 用户 sk。 -- `access_key` 碰撞时 MUST 重试生成,MUST NOT 写入半截配置。 - -### `grant` / `revoke` - -- `grant` 是累加:已授权的 library 保持不变,新名字追加。未知用户或未知 library MUST 失败,且 MUST NOT 改文件。 -- `revoke` 从该用户的 `datasets` 里去掉列出的名字。未知用户、或该用户当前并未持有的 library 名 MUST 失败。 -- 两个命令的 stdout 只报 `name` 与更新后的 `datasets`,MUST NOT 打印密钥。 - -改写配置可以整表重写,不要求保留注释。新用户在重启 serve 之前无法登录。 +`user create` 生成用户 AK/SK;secret 只在本次 stdout 输出。`dataset create` 只登记 Dataset,不创建或删除后端数据。`grant` 和 `revoke` 修改独立的 `[[grants]]` 授权记录。所有写操作 MUST 原子替换文件,失败时保留原文件。 ## HTTP @@ -270,7 +259,7 @@ Worker 用票构造 `ChronicleServerConfig` mounts,执行与普通 Warehouse 拒绝。第二套 listener、端口和生命周期会与 Warehouse 文档分叉。Catalog 目录流量很小,适合挂在现有 `pchronicle serve` 上。 -### 父进程打开全部 libraries 再按用户过滤 SQL +### 父进程打开全部 datasets 再按用户过滤 SQL 拒绝。DataFusion 与对象存储客户端一旦持有全量后端密钥和 mount,过滤错误就会越权。Web 查询必须在只含授权 mounts 的进程里执行。 diff --git a/justfile b/justfile index 836145e5..a9b738b8 100644 --- a/justfile +++ b/justfile @@ -64,6 +64,7 @@ test-list: just gateway-fuzz 一分钟 Gateway 四类 fuzz 汇总 just gateway-fuzz-formats / gateway-fuzz-forwarding just gateway-fuzz-storage / gateway-fuzz-network + just cases pvisor|pchronicle|pchronicle-cluster 组件示例 just examples-pvisor 全部 pVisor 场景 @@ -754,3 +755,40 @@ check-quick: # capture 相关 Rust 测试(Gateway 包测试已覆盖全部 capture targets)。 capture-test: just test-crate capture + +# Execute pChronicle single-machine/self-service cases. +[group('test')] +test-pchronicle-cases: + cargo build --release -p persisting-pchronicle-cli --locked + python3 scripts/run-pchronicle-cases.py --document docs/src/pchronicle/reference/cases-self.md --pchronicle target/release/pchronicle --report target/pchronicle-self-case-report.md + +# List and execute pChronicle platform/Catalog cases. Server lifecycle cases are +# reported as MANUAL unless explicitly selected with PCHRONICLE_CASE_MODE. +[group('test')] +test-pchronicle-cases-platform: + cargo build --release -p persisting-pchronicle-cli --locked + python3 scripts/run-pchronicle-cases.py --document docs/src/pchronicle/reference/cases-platform.md --pchronicle target/release/pchronicle --report target/pchronicle-platform-case-report.md + +# Run documented integration cases by component. +# Examples: just cases pvisor | pchronicle | pchronicle-cluster +cases target: + #!/usr/bin/env bash + set -euo pipefail + case "{{target}}" in + pvisor) + cargo build --release -p persisting-pvisor --locked + python3 scripts/run-pvisor-cases.py --report target/pvisor-case-report.md + ;; + pchronicle) + cargo build --release -p persisting-pchronicle-cli --locked + python3 scripts/run-pchronicle-cases.py --document docs/src/pchronicle/reference/cases-self.md --pchronicle target/release/pchronicle --report target/pchronicle-self-case-report.md + ;; + pchronicle-cluster) + cargo build --release -p persisting-pchronicle-cli --locked + python3 scripts/run-pchronicle-cases.py --document docs/src/pchronicle/reference/cases-platform.md --pchronicle target/release/pchronicle --report target/pchronicle-platform-case-report.md + ;; + *) + echo "usage: just cases pvisor|pchronicle|pchronicle-cluster" >&2 + exit 2 + ;; + esac diff --git a/scripts/run-pchronicle-cases.py b/scripts/run-pchronicle-cases.py new file mode 100755 index 00000000..fc681119 --- /dev/null +++ b/scripts/run-pchronicle-cases.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Run executable bash examples embedded in pChronicle cases documents.""" +from __future__ import annotations +import argparse, datetime as dt, os, re, subprocess, tempfile, time +from pathlib import Path + +CASE_RE=re.compile(r'^##\s+([SP]\d{2}):?\s*(.*)$') + +def parse(path): + lines=path.read_text(encoding='utf-8').splitlines(); out=[]; i=0 + while i