diff --git a/Cargo.lock b/Cargo.lock index 9c38a47e..22ef3ee2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7279,6 +7279,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "toml", "tracing", "url", "uuid", diff --git a/crates/persisting-pchronicle-cli/src/agent.rs b/crates/persisting-pchronicle-cli/src/agent.rs index 25bc6ab4..6d29a1bd 100644 --- a/crates/persisting-pchronicle-cli/src/agent.rs +++ b/crates/persisting-pchronicle-cli/src/agent.rs @@ -101,6 +101,12 @@ pub(super) struct AgentArgs { dry_run: bool, } +impl AgentArgs { + pub(super) fn dataset_reference(&self) -> Option<&str> { + self.dataset.as_deref().or(self.legacy_dataset.as_deref()) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] enum AgentTarget { Codex, diff --git a/crates/persisting-pchronicle-cli/src/lib.rs b/crates/persisting-pchronicle-cli/src/lib.rs index 7a0ea7b3..63275b42 100644 --- a/crates/persisting-pchronicle-cli/src/lib.rs +++ b/crates/persisting-pchronicle-cli/src/lib.rs @@ -123,6 +123,62 @@ impl Cli { } } +/// Apply S3 backend keys from `--catalog-config` and local `@alias` settings +/// before the multi-threaded Tokio runtime starts. `std::env::set_var` after +/// worker threads exist is racy on macOS and can leave OpenDAL unable to see +/// `AWS_REGION`. +pub fn apply_catalog_backend_env_before_runtime(cli: &Cli) -> Result<()> { + apply_serve_catalog_backend_env(cli)?; + apply_command_alias_backend_env(cli)?; + Ok(()) +} + +fn apply_serve_catalog_backend_env(cli: &Cli) -> Result<()> { + let Command::Serve(args) = &cli.command else { + return Ok(()); + }; + if args.command.is_some() || args.catalog_query_worker { + return Ok(()); + } + let Some(path) = args.catalog_config.as_ref() else { + return Ok(()); + }; + let acl = server::catalog::CatalogAcl::load(path)?; + acl.apply_backend_env(); + Ok(()) +} + +fn apply_command_alias_backend_env(cli: &Cli) -> Result<()> { + let reference = primary_dataset_reference(&cli.command); + apply_local_alias_backend_env_before_runtime(reference, cli.config.as_deref()) +} + +fn primary_dataset_reference(command: &Command) -> Option<&str> { + match command { + Command::Ls(args) => args.dataset_uri.as_deref(), + Command::Status(args) => args.dataset_uri.as_deref(), + Command::Query(args) => args.dataset_uri.as_deref(), + Command::Analysis(args) => match &args.command { + AnalysisCommand::Overview(options) + | AnalysisCommand::Agents(options) + | AnalysisCommand::Models(options) + | AnalysisCommand::Tools(options) => options.dataset_uri.as_deref(), + }, + Command::Find(args) => args.dataset_uri.as_deref(), + Command::Drop(args) => Some(args.dataset_uri.as_str()), + Command::Export(args) => args.from.as_deref(), + Command::Agent(args) => args.dataset_reference(), + Command::Import(args) => args.output.as_deref(), + Command::Onboard(_) + | Command::Default(_) + | Command::Alias(_) + | Command::Sync(_) + | Command::Echo(_) + | Command::Dev(_) + | Command::Serve(_) => None, + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] pub enum LogLevel { Error, @@ -207,7 +263,7 @@ enum Command { /// /// Compact JSONL stores each JSON object as one record without trajectory semantics. /// Select it with either --input-format compact-jsonl or - /// --output-format compact-jsonl; it accepts local .jsonl input and supports + /// --output-format compact-jsonl; it accepts local JSON/JSONL input and supports /// create or replace, not stdin or append. Import(ImportArgs), /// Permanently delete a Dataset directory or object-store prefix. @@ -226,7 +282,7 @@ enum Command { #[command(hide = true)] Dev(DevArgs), /// Run explicitly enabled Warehouse, Control, and Gateway services. - Serve(ServeArgs), + Serve(Box), } #[derive(Debug, Args)] @@ -702,7 +758,7 @@ struct ImportArgs { output: Option, /// Input exchange format. Auto detects run data; compact-jsonl is explicit. - /// Compact JSONL recursively reads local .jsonl files, one object per line. + /// Compact JSONL recursively reads local .json, .jsonl, or .ndjson files. #[arg(short = 'i', long = "input-format", alias = "format", value_enum, default_value_t = ExchangeFormat::Auto)] format: ExchangeFormat, @@ -730,7 +786,8 @@ struct ImportArgs { #[arg(long, value_parser = parse_byte_size, default_value = "256MiB")] max_input_bytes: Option, - /// Compact JSONL mapping. id/timestamp override $.id/$.timestamp; other names add JSONB columns. + /// Compact JSONL mapping. id/timestamp override $.id/$.timestamp; missing or invalid id values + /// use source_filename#line_number; other names add JSONB columns. /// Example: --column id=$.event.id --column model=$.payload.model. #[arg(long = "column", value_name = "NAME=JSON_PATH", action = clap::ArgAction::Append)] columns: Vec, @@ -978,6 +1035,62 @@ enum CatalogCommand { Grant(CatalogGrantArgs), /// Remove library names from a user's grants. Revoke(CatalogRevokeArgs), + /// Add, remove, or list datasets (libraries) in the catalog file. + Dataset(CatalogDatasetManageArgs), +} + +#[derive(Debug, Args)] +struct CatalogDatasetManageArgs { + #[command(subcommand)] + command: CatalogDatasetCommand, +} + +#[derive(Debug, Subcommand)] +enum CatalogDatasetCommand { + /// Register a dataset URI in the catalog file. + Add(CatalogDatasetAddArgs), + /// Remove dataset entries that are not referenced by grants. + Remove(CatalogDatasetRemoveArgs), + /// List datasets without printing backend secrets. + List(CatalogDatasetListArgs), +} + +#[derive(Debug, Args)] +struct CatalogDatasetAddArgs { + #[command(flatten)] + file: CatalogFileArg, + #[arg(value_name = "NAME")] + name: String, + #[arg(long = "uri", value_name = "URI")] + uri: String, + #[arg(long = "endpoint", value_name = "URL")] + endpoint: Option, + #[arg(long = "region", value_name = "REGION")] + region: Option, + #[arg(long = "access-key", value_name = "KEY")] + access_key: Option, + #[arg(long = "secret-key", value_name = "KEY")] + secret_key: Option, + #[arg(long, value_enum, default_value_t = OutputFormat::Auto)] + format: OutputFormat, +} + +#[derive(Debug, Args)] +struct CatalogDatasetRemoveArgs { + #[command(flatten)] + file: CatalogFileArg, + #[arg(value_name = "NAME", required = true, num_args = 1..)] + names: Vec, + #[arg(long, value_enum, default_value_t = OutputFormat::Auto)] + format: OutputFormat, +} + +#[derive(Debug, Args)] +struct CatalogDatasetListArgs { + #[command(flatten)] + file: CatalogFileArg, + #[arg(long, value_enum, default_value_t = OutputFormat::Auto)] + format: OutputFormat, } #[derive(Debug, Args)] @@ -1502,7 +1615,7 @@ pub async fn run_with_stdio( if let Some(command) = args.command { return run_serve_catalog(command, stdout_is_terminal, stdout, &mut diagnostics); } - run_serve(args, config, cli.log_level, stdout, &mut diagnostics).await + run_serve(*args, config, cli.log_level, stdout, &mut diagnostics).await } } } @@ -1944,6 +2057,45 @@ fn run_serve_catalog( &datasets, ) } + CatalogCommand::Dataset(dataset) => match dataset.command { + CatalogDatasetCommand::Add(add) => { + let library = server::catalog::add_dataset( + &add.file.catalog_config, + server::catalog::DatasetAddSpec { + name: add.name, + uri: add.uri, + endpoint: add.endpoint, + region: add.region, + access_key: add.access_key, + secret_key: add.secret_key, + }, + )?; + write_catalog_updated(stderr, &add.file.catalog_config)?; + write_catalog_dataset( + stdout, + resolve_output_format(add.format, stdout_is_terminal), + &library, + ) + } + CatalogDatasetCommand::Remove(remove) => { + let remaining = + server::catalog::remove_datasets(&remove.file.catalog_config, &remove.names)?; + write_catalog_updated(stderr, &remove.file.catalog_config)?; + write_catalog_dataset_names( + stdout, + resolve_output_format(remove.format, stdout_is_terminal), + &remaining, + ) + } + CatalogDatasetCommand::List(list) => { + let libraries = server::catalog::list_datasets_config(&list.file.catalog_config)?; + write_catalog_datasets( + stdout, + resolve_output_format(list.format, stdout_is_terminal), + &libraries, + ) + } + }, } } @@ -2009,6 +2161,70 @@ fn write_user_grants( } } +fn write_catalog_dataset( + stdout: &mut dyn Write, + format: OutputFormat, + library: &server::catalog::CatalogLibraryPublic, +) -> Result<()> { + match format { + OutputFormat::Table => { + writeln!(stdout, "NAME\tURI")?; + writeln!(stdout, "{}\t{}", library.name, library.uri) + .context("write catalog dataset table") + } + OutputFormat::Json => { + serde_json::to_writer(&mut *stdout, library).context("write catalog dataset JSON")?; + writeln!(stdout).context("finish catalog dataset JSON") + } + OutputFormat::Auto => unreachable!("auto output format was resolved"), + } +} + +fn write_catalog_datasets( + stdout: &mut dyn Write, + format: OutputFormat, + libraries: &[server::catalog::CatalogLibraryPublic], +) -> Result<()> { + match format { + OutputFormat::Table => { + writeln!(stdout, "NAME\tURI")?; + for library in libraries { + writeln!(stdout, "{}\t{}", library.name, library.uri) + .context("write catalog dataset row")?; + } + Ok(()) + } + OutputFormat::Json => { + serde_json::to_writer(&mut *stdout, libraries) + .context("write catalog datasets JSON")?; + writeln!(stdout).context("finish catalog datasets JSON") + } + OutputFormat::Auto => unreachable!("auto output format was resolved"), + } +} + +fn write_catalog_dataset_names( + stdout: &mut dyn Write, + format: OutputFormat, + names: &[String], +) -> Result<()> { + match format { + OutputFormat::Table => { + writeln!(stdout, "NAME")?; + for name in names { + writeln!(stdout, "{name}").context("write catalog dataset name")?; + } + Ok(()) + } + OutputFormat::Json => { + serde_json::to_writer(&mut *stdout, &serde_json::json!({ "datasets": names })) + .context("write catalog dataset names JSON")?; + writeln!(stdout).context("finish catalog dataset names JSON") + } + OutputFormat::Auto => unreachable!("auto output format was resolved"), + } +} + async fn run_serve( args: ServeArgs, settings_override: Option<&Path>, @@ -2023,7 +2239,18 @@ async fn run_serve( } let catalog_only = args.catalog_config.is_some(); let config = if catalog_only { - server::ChronicleServerConfig::front_only() + // Projection supervisor still needs the mount list; Warehouse prepare + // reloads the same catalog. Avoid front_only here so Gateway/Control + // siblings see the configured datasets. + let acl = server::catalog::CatalogAcl::load( + args.catalog_config + .as_ref() + .expect("catalog_only implies catalog_config"), + )?; + // OpenDAL/Lance read AWS_* from the process environment. Apply catalog + // backend keys before any discover/projection work touches s3:// mounts. + acl.apply_backend_env(); + server::ChronicleServerConfig::mounted(acl.mounts()?)? } else { resolve_serve_config_with_settings(&args, settings_override)? }; @@ -2059,7 +2286,7 @@ async fn run_serve( .with_context(|| format!("bind pChronicle Warehouse to {listen}"))?; let warehouse = if let Some(path) = args.catalog_config.as_ref() { let acl = server::catalog::CatalogAcl::load(path)?; - server::PreparedWarehouse::prepare_catalog_front(acl).await? + server::PreparedWarehouse::prepare_catalog(acl).await? } else if args.gateway.is_some() { server::PreparedWarehouse::prepare_live(config.clone()).await? } else { @@ -2438,6 +2665,24 @@ async fn run_list( stdout: &mut dyn Write, stderr: &mut dyn Write, ) -> Result<()> { + if let Some(reference) = args.dataset_uri.as_deref() { + let reference = reference.to_owned(); + let settings_path = settings_override.map(Path::to_path_buf); + let listing = tokio::task::spawn_blocking(move || { + list_catalog_alias_datasets(&reference, settings_path.as_deref()) + }) + .await + .context("list catalog alias datasets")??; + if let Some(listing) = listing { + return write_catalog_alias_dataset_list( + listing, + args.format, + stdout_is_terminal, + stdout, + stderr, + ); + } + } let dataset_uri = resolve_dataset_uri(args.dataset_uri.as_deref(), settings_override)?; let (dataset_uri, snapshot) = discover_snapshot(&dataset_uri, args.errors, args.max_files, args.max_entries).await?; @@ -2478,6 +2723,52 @@ async fn run_list( Ok(()) } +fn write_catalog_alias_dataset_list( + listing: CatalogAliasDatasetList, + format: OutputFormat, + stdout_is_terminal: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> Result<()> { + let output_format = match format { + OutputFormat::Auto if stdout_is_terminal => OutputFormat::Table, + OutputFormat::Auto => OutputFormat::Json, + explicit => explicit, + }; + match output_format { + OutputFormat::Table => { + writeln!(stdout, "DATASET\tURI\tENDPOINT\tREGION") + .context("write catalog ls header")?; + for dataset in &listing.datasets { + writeln!( + stdout, + "{}\t{}\t{}\t{}", + dataset.name, + dataset.uri, + dataset.endpoint.as_deref().unwrap_or("-"), + dataset.region.as_deref().unwrap_or("-"), + ) + .context("write catalog ls row")?; + } + } + OutputFormat::Json => { + serde_json::to_writer_pretty(&mut *stdout, &listing) + .context("encode catalog alias ls JSON")?; + writeln!(stdout).context("write catalog alias ls JSON")?; + } + OutputFormat::Auto => unreachable!("auto output format was resolved"), + } + writeln!( + stderr, + "alias={} catalog={} datasets={}", + listing.alias, + listing.catalog, + listing.datasets.len(), + ) + .context("write catalog alias ls metadata")?; + Ok(()) +} + fn source_response(source: &DiscoveredSource) -> SourceResponse { SourceResponse { source_path: source.file.clone(), diff --git a/crates/persisting-pchronicle-cli/src/main.rs b/crates/persisting-pchronicle-cli/src/main.rs index a83c00d5..413ce6b6 100644 --- a/crates/persisting-pchronicle-cli/src/main.rs +++ b/crates/persisting-pchronicle-cli/src/main.rs @@ -4,12 +4,42 @@ use std::io::{self, IsTerminal}; use std::process::ExitCode; use clap::Parser; -use persisting_pchronicle_cli::{Cli, error_code, error_exit_code, run_with_stdio}; +use persisting_pchronicle_cli::{ + Cli, apply_catalog_backend_env_before_runtime, error_code, error_exit_code, run_with_stdio, +}; -#[tokio::main] -async fn main() -> ExitCode { +fn main() -> ExitCode { let cli = Cli::parse(); let debug_errors = cli.debug_errors(); + // OpenDAL/Lance read AWS_* from the process environment. Applying catalog + // backend keys after the multi-threaded Tokio runtime starts is racy on + // macOS; do it before any worker threads exist. + if let Err(error) = apply_catalog_backend_env_before_runtime(&cli) { + use std::io::Write as _; + let code = error_code(&error); + let rendered = render_error(&error, debug_errors); + let _ = writeln!(io::stderr(), "error[{code}]: {rendered}"); + return ExitCode::from(error_exit_code(&error)); + } + + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + use std::io::Write as _; + let _ = writeln!( + io::stderr(), + "error[internal]: start tokio runtime: {error}" + ); + return ExitCode::from(1); + } + }; + runtime.block_on(async_main(cli, debug_errors)) +} + +async fn async_main(cli: Cli, debug_errors: bool) -> ExitCode { let stdin_is_terminal = io::stdin().is_terminal(); let stdout_is_terminal = io::stdout().is_terminal(); // Do not hold StdoutLock/StderrLock for the process lifetime. `pchronicle diff --git a/crates/persisting-pchronicle-cli/src/server/acceleration.rs b/crates/persisting-pchronicle-cli/src/server/acceleration.rs index b51a321b..b8e502cb 100644 --- a/crates/persisting-pchronicle-cli/src/server/acceleration.rs +++ b/crates/persisting-pchronicle-cli/src/server/acceleration.rs @@ -915,6 +915,12 @@ async fn build_run_summaries( .iter() .filter(|source| source.format.as_deref() == Some("compact-jsonl/v1")) { + // Manifest-backed compact sources are paged on demand by the runs + // API. Expanding hundreds of thousands of identities here freezes + // both Warehouse refresh and the WebAssembly client path_index. + if source.record_count.is_some() { + continue; + } if let Some(records) = snapshot.compact_records(name, &source.file).await? { for record in records { let path = explorer::explorer_run_path( @@ -939,6 +945,7 @@ async fn build_run_summaries( duplicate_event_ids: 0, status: "record".into(), format: Some("compact-jsonl/v1".into()), + explorer_weight: None, }); } } @@ -1016,6 +1023,7 @@ async fn build_run_summaries( duplicate_event_ids: event_stats.map_or(0, |stats| stats.duplicate_event_ids), status, format: None, + explorer_weight: None, }); } } diff --git a/crates/persisting-pchronicle-cli/src/server/catalog.rs b/crates/persisting-pchronicle-cli/src/server/catalog.rs index 44115d74..d3fefd83 100644 --- a/crates/persisting-pchronicle-cli/src/server/catalog.rs +++ b/crates/persisting-pchronicle-cli/src/server/catalog.rs @@ -106,10 +106,6 @@ impl CatalogAcl { pub(crate) fn parse(content: &str) -> Result { let file = parse_catalog_file(content)?; - anyhow::ensure!( - !file.users.is_empty(), - "catalog config needs at least one user" - ); Self::from_document(file) } @@ -121,6 +117,23 @@ impl CatalogAcl { }) } + pub(crate) fn mounts(&self) -> Result> { + self.libraries + .values() + .map(|library| DatasetMount::new(&library.name, &library.uri)) + .collect() + } + + pub(crate) fn apply_backend_env(&self) { + if let Some(library) = self + .libraries + .values() + .find(|library| library.access_key.is_some()) + { + apply_library_env(library); + } + } + pub(crate) fn authenticate(&self, access_key: &str, secret_key: &str) -> Option<&CatalogUser> { let user = self.users_by_access_key.get(access_key)?; if !secret_keys_match(&user.secret_key, secret_key) { @@ -233,6 +246,155 @@ pub(crate) fn revoke_datasets(path: &Path, name: &str, datasets: &[String]) -> R Ok(remaining) } +#[derive(Debug, Clone)] +pub(crate) struct DatasetAddSpec { + pub name: String, + pub uri: String, + pub endpoint: Option, + pub region: Option, + pub access_key: Option, + pub secret_key: Option, +} + +pub(crate) fn add_dataset(path: &Path, spec: DatasetAddSpec) -> Result { + let name = DatasetMount::new(&spec.name, "validation") + .with_context(|| format!("catalog dataset name '{}'", spec.name))? + .name; + anyhow::ensure!( + name == spec.name, + "catalog dataset '{}' must match [A-Za-z_][A-Za-z0-9_]* in lowercase", + spec.name + ); + let mut file = load_editable_catalog(path)?; + anyhow::ensure!( + !file.datasets.contains_key(&name), + "catalog dataset '{name}' already exists" + ); + let location = DatasetLocation::parse(&spec.uri) + .with_context(|| format!("catalog dataset '{name}' URI"))?; + let uri = location.as_str().to_owned(); + let is_s3 = uri.starts_with("s3://"); + let access_key = spec + .access_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned); + let secret_key = spec + .secret_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned); + match (access_key.as_ref(), secret_key.as_ref(), is_s3) { + (None, None, false) => {} + (Some(_), Some(_), true) => {} + (None, None, true) => anyhow::bail!( + "catalog dataset '{name}' is s3:// and must set --access-key and --secret-key" + ), + (Some(_), Some(_), false) => { + anyhow::bail!("catalog dataset '{name}' is not s3:// and must not set backend keys") + } + _ => anyhow::bail!("catalog dataset '{name}' must set both --access-key and --secret-key"), + } + if is_s3 { + ensure_s3_credentials_match_existing( + &file, + access_key.as_deref(), + secret_key.as_deref(), + spec.endpoint.as_deref(), + spec.region.as_deref(), + )?; + } + file.datasets.insert( + name.clone(), + CatalogLibraryFile { + uri: uri.clone(), + endpoint: spec.endpoint.clone(), + region: spec.region.clone(), + access_key, + secret_key, + }, + ); + // Validate full document before persist. + let _ = build_libraries(&file)?; + write_catalog_file(path, &file)?; + Ok(CatalogLibraryPublic { + name, + uri, + endpoint: spec.endpoint, + region: spec.region, + }) +} + +pub(crate) fn remove_datasets(path: &Path, names: &[String]) -> Result> { + let mut file = load_editable_catalog(path)?; + let library_names = canonical_library_names(&file)?; + let mut to_remove = Vec::new(); + for name in names { + let name = granted_library_name(&library_names, name)?; + let still_granted = file + .grants + .iter() + .filter(|grant| grant.dataset == name) + .map(|grant| grant.user.as_str()) + .collect::>(); + anyhow::ensure!( + still_granted.is_empty(), + "catalog dataset '{name}' is still granted to {}", + still_granted.join(", ") + ); + to_remove.push(name); + } + for name in &to_remove { + file.datasets.remove(name); + } + anyhow::ensure!( + !file.datasets.is_empty() || file.users.is_empty(), + "catalog config needs at least one dataset while users remain" + ); + if !file.datasets.is_empty() { + let _ = build_libraries(&file)?; + } + write_catalog_file(path, &file)?; + Ok(file.datasets.keys().cloned().collect()) +} + +pub(crate) fn list_datasets_config(path: &Path) -> Result> { + let file = load_editable_catalog(path)?; + if file.datasets.is_empty() { + return Ok(Vec::new()); + } + let libraries = build_libraries(&file)?; + Ok(libraries.values().map(CatalogLibraryPublic::from).collect()) +} + +fn ensure_s3_credentials_match_existing( + file: &CatalogFile, + access_key: Option<&str>, + secret_key: Option<&str>, + endpoint: Option<&str>, + region: Option<&str>, +) -> Result<()> { + for (name, library) in &file.datasets { + let uri = DatasetLocation::parse(&library.uri) + .with_context(|| format!("catalog library '{name}' URI"))? + .as_str() + .to_owned(); + if !uri.starts_with("s3://") { + continue; + } + anyhow::ensure!( + library.access_key.as_deref().map(str::trim) == access_key + && library.secret_key.as_deref().map(str::trim) == secret_key + && library.endpoint.as_deref() == endpoint + && library.region.as_deref() == region, + "catalog s3 datasets must share the same endpoint, region, and backend keys (differs from '{name}')" + ); + } + Ok(()) +} + fn read_catalog_config(path: &Path) -> Result { if !path.exists() { return Ok("[meta]\nversion = 1\nrevision = 0\n".to_owned()); @@ -455,7 +617,7 @@ fn encode_hex(bytes: &[u8]) -> String { encoded } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct CatalogLibraryPublic { pub name: String, pub uri: String, @@ -489,6 +651,7 @@ pub(crate) fn apply_library_env(library: &CatalogLibrary) { if let Some(region) = library.region.as_deref() { unsafe { std::env::set_var("AWS_REGION", region); + std::env::set_var("AWS_DEFAULT_REGION", region); } } if let (Some(access_key), Some(secret_key)) = @@ -634,7 +797,10 @@ pub(super) async fn catalog_data_plane_layer( ) -> axum::response::Response { use axum::response::IntoResponse; - if state.catalog_query_worker || state.catalog_acl.is_none() { + if state.catalog_query_worker + || state.catalog_acl.is_none() + || !state.config.datasets.is_empty() + { return next.run(request).await; } let path = request.uri().path().to_owned(); @@ -1087,6 +1253,37 @@ dataset = "prod" .router() } + #[tokio::test] + async fn prepare_catalog_mounts_local_datasets_without_credentials() { + let temporary = tempfile::tempdir().unwrap(); + let left = temporary.path().join("left"); + let right = temporary.path().join("right"); + std::fs::create_dir_all(&left).unwrap(); + std::fs::create_dir_all(&right).unwrap(); + let catalog = temporary.path().join("catalog.toml"); + std::fs::write( + &catalog, + format!( + r#" +[datasets.left] +uri = "{}" + +[datasets.right] +uri = "{}" +"#, + left.display(), + right.display() + ), + ) + .unwrap(); + + let acl = CatalogAcl::load(&catalog).unwrap(); + let warehouse = crate::server::PreparedWarehouse::prepare_catalog(acl) + .await + .unwrap(); + assert_eq!(warehouse.dataset_names(), vec!["left", "right"]); + } + async fn catalog_body(response: axum::response::Response) -> (axum::http::StatusCode, String) { use http_body_util::BodyExt; @@ -1268,9 +1465,131 @@ secret_key = "BACKEND_SK" } #[test] - fn parse_rejects_catalog_without_users() { - let error = CatalogAcl::parse(LIBRARIES_ONLY).unwrap_err().to_string(); - assert!(error.contains("at least one user"), "{error}"); + fn apply_catalog_backend_env_before_runtime_loads_s3_region() { + use clap::Parser; + + let temporary = tempfile::tempdir().unwrap(); + let catalog = temporary.path().join("catalog.toml"); + std::fs::write( + &catalog, + r#" +[datasets.rfs] +uri = "s3://test/test" +endpoint = "http://127.0.0.1:9000" +region = "us-east-1" +access_key = "123" +secret_key = "123" +"#, + ) + .unwrap(); + let catalog_arg = catalog.to_string_lossy().into_owned(); + let cli = crate::Cli::try_parse_from([ + "pchronicle", + "serve", + "--listen", + "127.0.0.1:0", + "--catalog-config", + &catalog_arg, + ]) + .unwrap(); + unsafe { + std::env::remove_var("AWS_REGION"); + std::env::remove_var("AWS_DEFAULT_REGION"); + } + crate::apply_catalog_backend_env_before_runtime(&cli).unwrap(); + assert_eq!(std::env::var("AWS_REGION").unwrap(), "us-east-1"); + assert_eq!(std::env::var("AWS_ACCESS_KEY_ID").unwrap(), "123"); + } + + #[test] + fn apply_library_env_exports_region_for_opendal() { + let previous_region = std::env::var("AWS_REGION").ok(); + let previous_default = std::env::var("AWS_DEFAULT_REGION").ok(); + unsafe { + std::env::remove_var("AWS_REGION"); + std::env::remove_var("AWS_DEFAULT_REGION"); + } + apply_library_env(&CatalogLibrary { + name: "rfs".into(), + uri: "s3://test/test".into(), + endpoint: Some("http://127.0.0.1:9000".into()), + region: Some("us-east-1".into()), + access_key: Some("123".into()), + secret_key: Some("123".into()), + }); + assert_eq!(std::env::var("AWS_REGION").unwrap(), "us-east-1"); + assert_eq!(std::env::var("AWS_DEFAULT_REGION").unwrap(), "us-east-1"); + assert_eq!(std::env::var("AWS_ACCESS_KEY_ID").unwrap(), "123"); + assert_eq!( + std::env::var("AWS_ENDPOINT_URL_S3").unwrap(), + "http://127.0.0.1:9000" + ); + unsafe { + match previous_region { + Some(value) => std::env::set_var("AWS_REGION", value), + None => std::env::remove_var("AWS_REGION"), + } + match previous_default { + Some(value) => std::env::set_var("AWS_DEFAULT_REGION", value), + None => std::env::remove_var("AWS_DEFAULT_REGION"), + } + } + } + + #[test] + fn parse_allows_catalog_without_users() { + let acl = CatalogAcl::parse(LIBRARIES_ONLY).unwrap(); + assert_eq!( + acl.mounts() + .unwrap() + .iter() + .map(|mount| mount.name.as_str()) + .collect::>(), + vec!["evals", "prod"] + ); + } + + #[test] + fn add_and_remove_dataset_rewrites_catalog_without_touching_users() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("catalog.toml"); + std::fs::write(&path, LIBRARIES_ONLY).unwrap(); + + let added = add_dataset( + &path, + DatasetAddSpec { + name: "local".into(), + uri: "/tmp/local-warehouse".into(), + endpoint: None, + region: None, + access_key: None, + secret_key: None, + }, + ) + .unwrap(); + assert_eq!(added.name, "local"); + assert_eq!(added.uri, "/tmp/local-warehouse"); + + let listed = list_datasets_config(&path).unwrap(); + assert!(listed.iter().any(|item| item.name == "local")); + + let remaining = remove_datasets(&path, &["local".into()]).unwrap(); + assert!(!remaining.iter().any(|name| name == "local")); + assert!(remaining.contains(&"prod".to_string())); + } + + #[test] + fn remove_dataset_rejects_when_grants_still_reference_it() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("catalog.toml"); + std::fs::write(&path, SAMPLE).unwrap(); + let error = remove_datasets(&path, &["prod".into()]) + .unwrap_err() + .to_string(); + assert!( + error.contains("still granted") || error.contains("grant"), + "{error}" + ); } #[test] diff --git a/crates/persisting-pchronicle-cli/src/server/explorer.rs b/crates/persisting-pchronicle-cli/src/server/explorer.rs index 3e8b14de..43450e5b 100644 --- a/crates/persisting-pchronicle-cli/src/server/explorer.rs +++ b/crates/persisting-pchronicle-cli/src/server/explorer.rs @@ -52,9 +52,12 @@ pub(crate) struct CatalogTree { pub(crate) struct CatalogTreeChild { pub(crate) name: String, pub(crate) kind: String, + pub(crate) data_type: String, pub(crate) path: String, pub(crate) run_count: usize, pub(crate) failed_count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) total_tokens: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) entries: Vec, } @@ -73,11 +76,17 @@ pub(crate) fn catalog_tree( && (dataset.is_none() || file_matches_prefix(&run.file, prefix)) }) .collect::>(); - let run_count = scoped.len(); + let run_count = scoped.iter().map(|run| explorer_weight(run)).sum(); let failed_count = scoped .iter() - .filter(|run| is_failed_status(&run.status)) - .count(); + .map(|run| { + if is_failed_status(&run.status) { + explorer_weight(run) + } else { + 0 + } + }) + .sum(); let children = if dataset.is_none() { fold_tree_children(dataset_children(&scoped), max_children, prefix) } else { @@ -97,6 +106,10 @@ fn is_failed_status(status: &str) -> bool { matches!(status, "failed" | "error") } +fn explorer_weight(run: &RunSummary) -> usize { + run.explorer_weight.unwrap_or(1).max(1) +} + fn file_matches_prefix(file: &str, prefix: &str) -> bool { prefix.is_empty() || file == prefix || file.starts_with(&format!("{prefix}/")) } @@ -105,6 +118,27 @@ struct ChildAcc { run_count: usize, failed_count: usize, has_deeper: bool, + data_types: BTreeSet, +} + +fn data_type(format: Option<&str>) -> &'static str { + match format { + Some("compact-jsonl/v1") => "compact-jsonl", + Some("storyline-lance") | None => "storyline", + Some(_) => "other", + } +} + +fn combined_data_type(data_types: &BTreeSet) -> String { + match data_types.len() { + 0 => "unknown".into(), + 1 => data_types + .iter() + .next() + .cloned() + .unwrap_or_else(|| "unknown".into()), + _ => "mixed".into(), + } } fn dataset_children(runs: &[&RunSummary]) -> Vec { @@ -114,10 +148,15 @@ fn dataset_children(runs: &[&RunSummary]) -> Vec { run_count: 0, failed_count: 0, has_deeper: false, + data_types: BTreeSet::new(), }); - entry.run_count += 1; + let weight = explorer_weight(run); + entry.run_count += weight; + entry + .data_types + .insert(data_type(run.format.as_deref()).into()); if is_failed_status(&run.status) { - entry.failed_count += 1; + entry.failed_count += weight; } } groups @@ -125,9 +164,11 @@ fn dataset_children(runs: &[&RunSummary]) -> Vec { .map(|(name, acc)| CatalogTreeChild { name: name.clone(), kind: "dataset".into(), + data_type: combined_data_type(&acc.data_types), path: name, run_count: acc.run_count, failed_count: acc.failed_count, + total_tokens: None, entries: Vec::new(), }) .collect() @@ -161,10 +202,15 @@ fn file_children(runs: &[&RunSummary], prefix: &str) -> Vec { run_count: 0, failed_count: 0, has_deeper: false, + data_types: BTreeSet::new(), }); - entry.run_count += 1; + let weight = explorer_weight(run); + entry.run_count += weight; + entry + .data_types + .insert(data_type(run.format.as_deref()).into()); if is_failed_status(&run.status) { - entry.failed_count += 1; + entry.failed_count += weight; } entry.has_deeper |= has_deeper; } @@ -184,6 +230,8 @@ fn file_children(runs: &[&RunSummary], prefix: &str) -> Vec { name, run_count: acc.run_count, failed_count: acc.failed_count, + data_type: combined_data_type(&acc.data_types), + total_tokens: None, entries: Vec::new(), }) .collect() @@ -208,14 +256,40 @@ fn fold_tree_children( children.push(CatalogTreeChild { name: "other".into(), kind: "other".into(), + data_type: "mixed".into(), path: prefix.to_string(), run_count: rest.iter().map(|child| child.run_count).sum(), failed_count: rest.iter().map(|child| child.failed_count).sum(), + total_tokens: None, entries: rest, }); children } +pub(crate) fn apply_total_tokens( + children: &mut [CatalogTreeChild], + file_tokens: &BTreeMap, +) { + for child in children { + apply_total_tokens(&mut child.entries, file_tokens); + child.total_tokens = if child.entries.is_empty() { + file_tokens + .iter() + .filter(|(file, _)| { + *file == &child.path || file.starts_with(&format!("{}/", child.path)) + }) + .map(|(_, tokens)| *tokens) + .reduce(u64::saturating_add) + } else { + child + .entries + .iter() + .filter_map(|entry| entry.total_tokens) + .reduce(u64::saturating_add) + }; + } +} + #[derive(Clone, Debug, Serialize)] pub(crate) struct ExplorerPage { pub(crate) snapshot: PageSnapshot, @@ -436,7 +510,37 @@ pub(crate) fn run_page_with_fts( ) }) .collect::>(); - let path_index = records.iter().map(|item| item.run.clone()).collect(); + let path_index_limit = 2_000usize; + let path_index = if records.len() > path_index_limit { + // Huge compact-jsonl sources must not ship every identity into the + // browser path explorer. Keep one representative per file plus a + // bounded sample so WASM stays responsive. + let mut index = Vec::with_capacity(path_index_limit); + let mut seen_files = BTreeSet::new(); + for item in &records { + if seen_files.insert(item.run.file.clone()) { + index.push(item.run.clone()); + } + if index.len() >= path_index_limit { + break; + } + } + for item in records + .iter() + .take(path_index_limit.saturating_sub(index.len())) + { + if index.iter().any(|run| run.path == item.run.path) { + continue; + } + index.push(item.run.clone()); + if index.len() >= path_index_limit { + break; + } + } + index + } else { + records.iter().map(|item| item.run.clone()).collect() + }; if let Some(path) = query .path .as_deref() @@ -1310,6 +1414,16 @@ mod tests { } fn sample_run(dataset: &str, file: &str, status: &str, session: &str) -> RunSummary { + sample_weighted_run(dataset, file, status, session, None) + } + + fn sample_weighted_run( + dataset: &str, + file: &str, + status: &str, + session: &str, + explorer_weight: Option, + ) -> RunSummary { RunSummary { dataset: dataset.into(), file: file.into(), @@ -1320,13 +1434,52 @@ mod tests { session_id: session.into(), root_session_id: None, path: format!("{dataset}/{file}/{session}"), - row_count: 1, + row_count: explorer_weight.unwrap_or(1), duplicate_event_ids: 0, status: status.into(), - format: None, + format: Some("compact-jsonl/v1".into()), + explorer_weight, } } + #[test] + fn nested_manifest_leaf_weights_roll_up_to_ancestor_prefixes() { + let runs = [ + sample_weighted_run("default", "archive", "record", "archive", Some(7)), + sample_weighted_run( + "default", + "team/codex_jsonl", + "record", + "codex_jsonl", + Some(10), + ), + sample_weighted_run("default", "team/evals", "record", "evals", Some(20)), + ]; + + let root = catalog_tree(&runs, Some("default"), "", 16); + assert_eq!(root.run_count, 37); + let children: Vec<_> = root + .children + .iter() + .map(|child| (child.name.as_str(), child.kind.as_str(), child.run_count)) + .collect(); + assert_eq!(children, vec![("team", "dir", 30), ("archive", "file", 7)]); + + let team = catalog_tree(&runs, Some("default"), "team", 16); + assert_eq!(team.run_count, 30); + assert_eq!( + team.children + .iter() + .map(|child| (child.name.as_str(), child.run_count)) + .collect::>(), + vec![("evals", 20), ("codex_jsonl", 10)] + ); + + let leaf = catalog_tree(&runs, Some("default"), "team/codex_jsonl", 16); + assert_eq!(leaf.run_count, 10); + assert!(leaf.children.is_empty()); + } + #[test] fn warehouse_tree_sizes_datasets_by_run_count() { let tree = catalog_tree( diff --git a/crates/persisting-pchronicle-cli/src/server/mod.rs b/crates/persisting-pchronicle-cli/src/server/mod.rs index 865f135d..b5d083ff 100644 --- a/crates/persisting-pchronicle-cli/src/server/mod.rs +++ b/crates/persisting-pchronicle-cli/src/server/mod.rs @@ -124,6 +124,10 @@ pub(crate) struct RunSummary { pub(crate) status: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) format: Option, + /// When set, explorer tree counts this summary as `explorer_weight` runs + /// instead of 1. Used for compact-jsonl leaves backed by chronicle.manifest. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) explorer_weight: Option, } #[derive(Debug, Clone, Deserialize)] @@ -185,12 +189,34 @@ impl PreparedWarehouse { Ok(warehouse) } + /// Directory front-only mode: parent authenticates and dispatches query + /// workers. Retained for isolation tests; `serve --catalog-config` uses + /// [`Self::prepare_catalog`] (inline mounts) instead. + #[cfg_attr(not(test), allow(dead_code))] pub(crate) async fn prepare_catalog_front(acl: catalog::CatalogAcl) -> anyhow::Result { let mut state = app_state(ChronicleServerConfig::front_only()); state.catalog_acl = Some(Arc::new(acl)); Ok(Self { state }) } + /// Mount every library from `catalog.toml` into the Warehouse process. + /// Directory ticket routes remain available when users exist; the data + /// plane serves in-process mounts instead of spawning query workers. + pub(crate) async fn prepare_catalog(acl: catalog::CatalogAcl) -> anyhow::Result { + acl.apply_backend_env(); + let mounts = acl.mounts()?; + anyhow::ensure!( + !mounts.is_empty(), + "catalog config needs at least one dataset" + ); + let config = ChronicleServerConfig::mounted(mounts)?; + let mut state = app_state(config); + state.catalog_acl = Some(Arc::new(acl)); + let warehouse = Self { state }; + warehouse.install_initial_runtime().await?; + Ok(warehouse) + } + pub(crate) async fn prepare_query_worker( config: ChronicleServerConfig, ) -> anyhow::Result { @@ -569,6 +595,138 @@ async fn explorer_query_jsonl( }) } +/// Serve manifesto-backed compact-jsonl leaves as a true page, instead of +/// expanding every record identity into `run_summaries` (which freezes WASM). +async fn try_compact_jsonl_runs_page( + state: &AppState, + query: &explorer::ExplorerRunsQuery, + request_id: &RequestId, +) -> Result, ApiError> { + if query + .q + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + { + return Ok(None); + } + let file_filter = query + .file + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); + let Some(file_filter) = file_filter else { + return Ok(None); + }; + let dataset_filter = query + .dataset + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty() && *value != "all"); + + let runtime = current_catalog(state, request_id).await?; + let mut matched: Option<(String, String, Option)> = None; + for dataset in runtime.snapshot.datasets() { + if dataset_filter.is_some_and(|filter| dataset.mount.name != filter) { + continue; + } + for source in &dataset.sources { + if source.format.as_deref() != Some("compact-jsonl/v1") { + continue; + } + if source.record_count.is_none() { + continue; + } + if source.file != file_filter && !source.file.starts_with(&format!("{file_filter}/")) { + continue; + } + if matched.is_some() { + // Ambiguous prefix across multiple compact leaves: fall back. + return Ok(None); + } + matched = Some(( + dataset.mount.name.clone(), + source.file.clone(), + source.record_count, + )); + } + } + let Some((dataset_name, source_file, record_count)) = matched else { + return Ok(None); + }; + + let offset = query.offset.unwrap_or(0); + let limit = query.limit.unwrap_or(50).clamp(1, 200); + let (records, total) = runtime + .snapshot + .compact_records_page(&dataset_name, &source_file, offset, limit) + .await + .map_err(|error| fail(request_id, "explorer_runs", error))? + .ok_or_else(|| { + fail( + request_id, + "explorer_runs", + anyhow::anyhow!("compact-jsonl source `{source_file}` is not resolvable"), + ) + })?; + let total = record_count.unwrap_or(total); + let total_usize = usize::try_from(total).unwrap_or(usize::MAX); + + let page_records = records + .into_iter() + .map(|record| { + let path = explorer::explorer_run_path( + &dataset_name, + &source_file, + &record.id, + &record.id, + None, + None, + ); + explorer::RunExplorerItem { + model: None, + search_preview: None, + run: RunSummary { + dataset: dataset_name.clone(), + file: source_file.clone(), + document_id: record.id.clone(), + run_id: None, + agent_id: "compact-jsonl".into(), + model_name: None, + session_id: record.id, + root_session_id: None, + path, + row_count: 1, + duplicate_event_ids: 0, + status: "record".into(), + format: Some("compact-jsonl/v1".into()), + explorer_weight: None, + }, + } + }) + .collect::>(); + + let next_offset = offset.saturating_add(page_records.len()); + let path_index = page_records + .iter() + .map(|item| item.run.clone()) + .collect::>(); + + Ok(Some(explorer::RunExplorerPage { + snapshot: explorer::PageSnapshot { + offset, + next_offset, + total: total_usize, + has_more: next_offset < total_usize, + limit, + }, + records: page_records, + // Mirror the current page into PathExplorer. Shipping every identity + // freezes WASM; an empty/stub-only index leaves the left nav blank. + path_index, + search: explorer::RunSearchStatus::default(), + })) +} + async fn explorer_runs( State(state): State, request_id: RequestId, @@ -576,6 +734,9 @@ async fn explorer_runs( query: Result, QueryRejection>, ) -> Result, ApiError> { let query = api_query(query)?; + if let Some(page) = try_compact_jsonl_runs_page(&state, &query, &request_id).await? { + return Ok(Json(page)); + } let dataset_filter = query .dataset .as_deref() @@ -816,16 +977,8 @@ async fn explorer_tree( query: Result, QueryRejection>, ) -> Result, ApiError> { let query = api_query(query)?; - // Tree navigation is a read of the already-installed catalog. Do not run - // the five-second automatic catalog refresh on every folder interaction; - // the runs endpoint remains the freshness boundary for live summaries. - let runtime = current_catalog(&state, &request_id).await?; - let summaries = runtime - .acceleration - .run_summaries(&runtime.snapshot, &runtime.engine) - .await - .map(|summaries| summaries.as_ref().clone()) - .map_err(|error| fail(&request_id, "explorer_tree", error))?; + let runtime = current_catalog_for_runs(&state, &request_id).await?; + let summaries = tree_run_summaries(&runtime, &request_id).await?; let dataset = query .dataset .as_deref() @@ -843,10 +996,75 @@ async fn explorer_tree( let (duration_ms, total_tokens) = tree_prefix_metrics(&runtime, &name, &tree.prefix).await; tree.duration_ms = duration_ms; tree.total_tokens = total_tokens; + let file_tokens = tree_file_tokens(&runtime, &name, &tree.prefix).await; + explorer::apply_total_tokens(&mut tree.children, &file_tokens); } Ok(Json(tree)) } +async fn tree_run_summaries( + runtime: &CatalogRuntime, + request_id: &RequestId, +) -> Result, ApiError> { + let mut summaries = Vec::new(); + let mut compact_with_manifest = BTreeSet::new(); + for dataset in runtime.snapshot.datasets() { + for source in &dataset.sources { + if source.format.as_deref() != Some("compact-jsonl/v1") { + continue; + } + let Some(record_count) = source.record_count else { + continue; + }; + let weight = usize::try_from(record_count).unwrap_or(usize::MAX); + let path = explorer::explorer_run_path( + &dataset.mount.name, + &source.file, + "", + &source.file, + None, + None, + ); + summaries.push(RunSummary { + dataset: dataset.mount.name.clone(), + file: source.file.clone(), + document_id: String::new(), + run_id: None, + agent_id: "compact-jsonl".into(), + model_name: None, + session_id: source.file.clone(), + root_session_id: None, + path, + row_count: weight, + duplicate_event_ids: 0, + status: "record".into(), + format: source.format.clone(), + explorer_weight: Some(weight.max(1)), + }); + compact_with_manifest.insert((dataset.mount.name.clone(), source.file.clone())); + } + } + if !runtime.snapshot.datasets().iter().any(|dataset| { + dataset.sources.iter().any(|source| { + source.format.as_deref() != Some("compact-jsonl/v1") || source.record_count.is_none() + }) + }) { + return Ok(summaries); + } + let full = runtime + .acceleration + .run_summaries(&runtime.snapshot, &runtime.engine) + .await + .map_err(|error| fail(request_id, "explorer_tree", error))?; + for summary in full.iter() { + if compact_with_manifest.contains(&(summary.dataset.clone(), summary.file.clone())) { + continue; + } + summaries.push(summary.clone()); + } + Ok(summaries) +} + fn sql_ident(name: &str) -> Option<&str> { let mut chars = name.chars(); let first = chars.next()?; @@ -870,7 +1088,11 @@ async fn tree_prefix_metrics( format!(" WHERE _file_ = '{escaped}' OR _file_ LIKE '{escaped}/%'") }; let sql = format!( - "SELECT MIN(timestamp) AS start_ts, MAX(timestamp) AS end_ts FROM {ident}.steps{file_clause}" + "SELECT MIN(timestamp) AS start_ts, MAX(timestamp) AS end_ts, SUM(COALESCE(\ + json_get_int(metrics, 'total_tokens'),\ + json_get_int(metrics, 'prompt_tokens') + json_get_int(metrics, 'completion_tokens'),\ + json_get_int(metrics, 'prompt_tokens_len') + json_get_int(metrics, 'completion_tokens_len')\ + )) AS total_tokens FROM {ident}.steps{file_clause}" ); let mut buffer = Vec::new(); let write = tokio::time::timeout( @@ -894,6 +1116,56 @@ async fn tree_prefix_metrics( ) } +async fn tree_file_tokens( + runtime: &CatalogRuntime, + dataset: &str, + prefix: &str, +) -> BTreeMap { + let Some(ident) = sql_ident(dataset) else { + return BTreeMap::new(); + }; + let file_clause = if prefix.is_empty() { + String::new() + } else { + let escaped = prefix.replace('\'', "''"); + format!(" WHERE _file_ = '{escaped}' OR _file_ LIKE '{escaped}/%'") + }; + let sql = format!( + "SELECT _file_, SUM(COALESCE(\ + json_get_int(metrics, 'total_tokens'),\ + json_get_int(metrics, 'prompt_tokens') + json_get_int(metrics, 'completion_tokens'),\ + json_get_int(metrics, 'prompt_tokens_len') + json_get_int(metrics, 'completion_tokens_len')\ + )) AS total_tokens FROM {ident}.steps{file_clause} GROUP BY _file_" + ); + let mut buffer = Vec::new(); + let write = tokio::time::timeout( + Duration::from_secs(3), + runtime + .engine + .write_query_jsonl_with_max_rows(&sql, &mut buffer, None), + ) + .await; + let Ok(Ok(())) = write else { + return BTreeMap::new(); + }; + buffer + .split(|byte| *byte == b'\n') + .filter_map(|line| { + let row: Value = serde_json::from_slice(line).ok()?; + let file = row.get("_file_")?.as_str()?.to_owned(); + let tokens = row + .get("total_tokens") + .and_then(Value::as_u64) + .or_else(|| { + row.get("total_tokens") + .and_then(Value::as_i64) + .map(|value| value.max(0) as u64) + })?; + Some((file, tokens)) + }) + .collect() +} + fn timestamp_span_ms(start: Option<&Value>, end: Option<&Value>) -> Option { let start = json_timestamp_ms(start?)?; let end = json_timestamp_ms(end?)?; diff --git a/crates/persisting-pchronicle-cli/src/server/tests.rs b/crates/persisting-pchronicle-cli/src/server/tests.rs index 7f85d8b2..8355bb53 100644 --- a/crates/persisting-pchronicle-cli/src/server/tests.rs +++ b/crates/persisting-pchronicle-cli/src/server/tests.rs @@ -826,6 +826,7 @@ fn explorer_analysis_counts_usage_and_normalized_tools_once_per_call() { duplicate_event_ids: 0, status: "completed".into(), format: None, + explorer_weight: None, }; let analysis = explorer::analyze(run, &turns, &events, CatalogEventProvenance::Canonical); @@ -872,6 +873,7 @@ fn canonical_event_uri_resolves_write_coordinates_independent_of_mount_root() { duplicate_event_ids: 0, status: "active".into(), format: None, + explorer_weight: None, }; let local = event_uri_coords("/tmp/capture/agent/run-1/events.lance", &run).unwrap(); assert_eq!(local.storage, "/tmp/capture"); @@ -991,6 +993,23 @@ async fn explorer_automatically_refreshes_new_dataset_sources() { assert_eq!(initial["snapshot"]["total"], 1); write_gateway_fixture(&root, "second.json", "second-session", "second-job"); + let tree = app + .clone() + .oneshot( + axum::http::Request::builder() + .uri(format!( + "/api/explorer/tree?dataset={}", + encode_query(DEFAULT_DATASET_NAME) + )) + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let tree: Value = + serde_json::from_slice(&tree.into_body().collect().await.unwrap().to_bytes()).unwrap(); + assert_eq!(tree["run_count"], 2); + let refreshed = app .oneshot( axum::http::Request::builder() @@ -1013,6 +1032,77 @@ async fn explorer_automatically_refreshes_new_dataset_sources() { std::fs::remove_dir_all(root).unwrap(); } +#[tokio::test] +async fn explorer_pages_manifest_backed_compact_jsonl_without_expanding_all_records() { + use http_body_util::BodyExt; + use tower::ServiceExt; + + let temp = tempfile::tempdir().unwrap(); + let input = temp.path().join("input.jsonl"); + let compact = temp.path().join("codex_jsonl"); + let mut body = String::new(); + for index in 0..5 { + body.push_str(&format!( + "{{\"timestamp\":{index},\"value\":\"row-{index}\"}}\n" + )); + } + std::fs::write(&input, body).unwrap(); + persisting_pchronicle::storage::CompactJsonlStore::import_path( + &input, + &compact, + &persisting_pchronicle::storage::CompactJsonlOptions::default(), + ) + .await + .unwrap(); + std::fs::remove_file(input).unwrap(); + + let app = router(temp.path().to_string_lossy().to_string()); + let response = app + .clone() + .oneshot( + axum::http::Request::builder() + .uri(format!( + "/api/explorer/runs?dataset={}&file=codex_jsonl&limit=2&offset=0", + encode_query(DEFAULT_DATASET_NAME) + )) + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let page: Value = + serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap(); + assert_eq!(page["snapshot"]["total"], 5); + assert_eq!(page["snapshot"]["limit"], 2); + assert_eq!(page["snapshot"]["has_more"], true); + assert_eq!(page["records"].as_array().unwrap().len(), 2); + assert_eq!(page["path_index"].as_array().unwrap().len(), 2); + assert_eq!(page["path_index"][0]["file"], "codex_jsonl"); + assert_eq!( + page["path_index"][0]["session_id"], + page["records"][0]["session_id"] + ); + + let page2 = app + .oneshot( + axum::http::Request::builder() + .uri(format!( + "/api/explorer/runs?dataset={}&file=codex_jsonl&limit=2&offset=4", + encode_query(DEFAULT_DATASET_NAME) + )) + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let page2: Value = + serde_json::from_slice(&page2.into_body().collect().await.unwrap().to_bytes()).unwrap(); + assert_eq!(page2["snapshot"]["total"], 5); + assert_eq!(page2["records"].as_array().unwrap().len(), 1); + assert_eq!(page2["snapshot"]["has_more"], false); +} + #[tokio::test] async fn server_routing_index_prunes_point_queries_and_resets_on_refresh() -> anyhow::Result<()> { use http_body_util::BodyExt; diff --git a/crates/persisting-pchronicle-cli/src/settings.rs b/crates/persisting-pchronicle-cli/src/settings.rs index d2482944..d44c9f4e 100644 --- a/crates/persisting-pchronicle-cli/src/settings.rs +++ b/crates/persisting-pchronicle-cli/src/settings.rs @@ -594,13 +594,52 @@ fn apply_alias_endpoint(settings: &LocalSettings, name: &str) { } } -fn apply_alias_region(settings: &LocalSettings, name: &str) { - let Some(region) = settings.alias_regions.get(name) else { - return; +const DEFAULT_S3_ALIAS_REGION: &str = "us-west-2"; + +fn apply_alias_region(settings: &LocalSettings, name: &str, s3_uri: bool) { + let region = match settings.alias_regions.get(name) { + Some(region) => region.as_str(), + // Documented fallback when an s3:// alias omits --region. OpenDAL + // requires AWS_REGION or AWS_DEFAULT_REGION at Builder::build. + None if s3_uri => DEFAULT_S3_ALIAS_REGION, + None => return, }; unsafe { std::env::set_var("AWS_REGION", region); + std::env::set_var("AWS_DEFAULT_REGION", region); + } +} + +/// Apply local `@alias` S3 backend keys before the multi-threaded Tokio runtime +/// starts. Same macOS `set_var` race as catalog serve: OpenDAL must see +/// `AWS_REGION` before worker threads exist. +pub(super) fn apply_local_alias_backend_env_before_runtime( + reference: Option<&str>, + settings_override: Option<&Path>, +) -> Result<()> { + let Some(reference) = reference.map(str::trim).filter(|value| !value.is_empty()) else { + return Ok(()); + }; + let Some(rest) = reference.strip_prefix('@') else { + return Ok(()); + }; + let (name, _) = rest.split_once('/').unwrap_or((rest, "")); + if name.is_empty() || RESERVED_ALIASES.contains(&name) { + return Ok(()); } + validate_alias_name(name)?; + let path = settings_path(settings_override)?; + let settings = load_local_settings_or_default(&path)?; + let Some(root) = settings.aliases.get(name) else { + return Ok(()); + }; + if !root.starts_with("s3://") { + return Ok(()); + } + apply_alias_credentials(&settings, name); + apply_alias_endpoint(&settings, name); + apply_alias_region(&settings, name, true); + Ok(()) } fn expand_catalog_alias( @@ -648,12 +687,109 @@ fn expand_catalog_alias( Ok(location.as_str().to_owned()) } +/// When `input` is a bare catalog alias (`@team` / `@team/`), return the alias +/// name and Directory URL. Dataset-qualified refs (`@team/prod`) return `None`. +pub(super) fn catalog_alias_directory_target( + input: &str, + settings_override: Option<&Path>, +) -> Result> { + let input = input.trim(); + let Some(rest) = input.strip_prefix('@') else { + return Ok(None); + }; + let (name, suffix) = rest.split_once('/').unwrap_or((rest, "")); + if !suffix.is_empty() || RESERVED_ALIASES.contains(&name) { + return Ok(None); + } + validate_alias_name(name)?; + let path = settings_path(settings_override)?; + let settings = load_local_settings_or_default(&path)?; + let root = settings.aliases.get(name).ok_or_else(|| { + cli_boundary_error( + BoundaryCode::NotFound, + format!("unknown Dataset alias '@{name}'"), + ) + })?; + if !root.starts_with("catalog://") { + return Ok(None); + } + Ok(Some((name.to_owned(), root.clone()))) +} + +/// List libraries visible to a catalog alias's user credentials. +pub(super) fn list_catalog_alias_datasets( + input: &str, + settings_override: Option<&Path>, +) -> Result> { + let Some((alias, catalog_url)) = catalog_alias_directory_target(input, settings_override)? + else { + return Ok(None); + }; + let path = settings_path(settings_override)?; + let settings = load_local_settings_or_default(&path)?; + let credentials = settings.alias_credentials.get(&alias).ok_or_else(|| { + cli_boundary_error( + BoundaryCode::InvalidRequest, + format!("catalog alias '@{alias}' requires --ak and --sk"), + ) + })?; + let datasets = fetch_catalog_datasets( + &catalog_url, + &credentials.access_key, + &credentials.secret_key, + )?; + Ok(Some(CatalogAliasDatasetList { + alias: format!("@{alias}"), + catalog: catalog_url, + datasets, + })) +} + +#[derive(Debug, Clone, Serialize)] +pub(super) struct CatalogAliasDatasetList { + pub alias: String, + pub catalog: String, + pub datasets: Vec, +} + thread_local! { static CATALOG_TICKETS: std::cell::RefCell< HashMap<(String, String, String), crate::server::catalog::CatalogLibrary>, > = std::cell::RefCell::new(HashMap::new()); } +fn fetch_catalog_datasets( + catalog_url: &str, + access_key: &str, + secret_key: &str, +) -> Result> { + use crate::server::catalog::{ACCESS_KEY_HEADER, SECRET_KEY_HEADER, catalog_http_base}; + + let base = catalog_http_base(catalog_url)?; + let url = format!("{base}/api/v1/catalog/datasets"); + let response = reqwest::blocking::Client::new() + .get(&url) + .header(ACCESS_KEY_HEADER, access_key) + .header(SECRET_KEY_HEADER, secret_key) + .send() + .with_context(|| format!("list catalog datasets at {catalog_url}"))?; + let status = response.status(); + let body = response + .text() + .context("read catalog dataset list response")?; + if !status.is_success() { + return Err(cli_boundary_error( + if status.as_u16() == 401 { + BoundaryCode::InvalidRequest + } else { + BoundaryCode::Unavailable + }, + format!("catalog dataset list failed ({status}): {body}"), + )); + } + serde_json::from_str(&body).context("decode catalog dataset list") +} + fn fetch_catalog_ticket( catalog_url: &str, access_key: &str, @@ -747,7 +883,7 @@ pub(super) fn expand_dataset_reference( let expanded = join_alias_target(root, suffix)?; apply_alias_credentials(&settings, name); apply_alias_endpoint(&settings, name); - apply_alias_region(&settings, name); + apply_alias_region(&settings, name, root.starts_with("s3://")); expanded } } diff --git a/crates/persisting-pchronicle-cli/src/tests.rs b/crates/persisting-pchronicle-cli/src/tests.rs index 1b0a3e65..9f9d3bed 100644 --- a/crates/persisting-pchronicle-cli/src/tests.rs +++ b/crates/persisting-pchronicle-cli/src/tests.rs @@ -319,7 +319,7 @@ fn command_tree_contains_the_product_commands() { .get_subcommands() .map(|command| command.get_name()) .collect::>(); - assert_eq!(catalog_commands, ["issue", "grant", "revoke"]); + assert_eq!(catalog_commands, ["issue", "grant", "revoke", "dataset"]); let mut serve_command = Cli::command(); let serve_help = serve_command.find_subcommand_mut("serve").unwrap(); let mut help = Vec::new(); @@ -547,6 +547,7 @@ async fn alias_s3_credentials_are_stored_separately_and_applied_on_expansion() - let _generic_endpoint = EnvGuard::unset("AWS_ENDPOINT"); let _allow_http = EnvGuard::unset("AWS_ALLOW_HTTP"); let _region = EnvGuard::unset("AWS_REGION"); + let _default_region = EnvGuard::unset("AWS_DEFAULT_REGION"); assert_eq!( expand_dataset_reference("@prod", Some(&config), false)?, "s3://example-bucket/evals" @@ -569,6 +570,10 @@ async fn alias_s3_credentials_are_stored_separately_and_applied_on_expansion() - ); assert_eq!(std::env::var("AWS_ALLOW_HTTP").as_deref(), Ok("true")); assert_eq!(std::env::var("AWS_REGION").as_deref(), Ok("us-west-2")); + assert_eq!( + std::env::var("AWS_DEFAULT_REGION").as_deref(), + Ok("us-west-2") + ); let cli = Cli::try_parse_from([ "pchronicle", @@ -587,6 +592,59 @@ async fn alias_s3_credentials_are_stored_separately_and_applied_on_expansion() - Ok(()) } +#[tokio::test] +async fn s3_alias_without_region_falls_back_to_documented_default() -> Result<()> { + let _env_guard = DATASET_ALIAS_ENV_LOCK.lock().await; + let temporary = tempfile::tempdir()?; + let config = temporary.path().join("config.toml"); + let config_arg = config.to_string_lossy().into_owned(); + let cli = Cli::try_parse_from([ + "pchronicle", + "-c", + &config_arg, + "alias", + "add", + "minio", + "s3://test/test", + "--endpoint", + "http://127.0.0.1:9000", + "--ak", + "123", + "--sk", + "123", + ])?; + run(cli, false, &mut Vec::new(), &mut Vec::new()).await?; + let config_text = fs::read_to_string(&config)?; + assert!(!config_text.contains("[alias_regions]"), "{config_text}"); + + let _region = EnvGuard::unset("AWS_REGION"); + let _default_region = EnvGuard::unset("AWS_DEFAULT_REGION"); + let _endpoint = EnvGuard::unset("AWS_ENDPOINT_URL_S3"); + assert_eq!( + expand_dataset_reference("@minio", Some(&config), false)?, + "s3://test/test" + ); + assert_eq!(std::env::var("AWS_REGION").as_deref(), Ok("us-west-2")); + assert_eq!( + std::env::var("AWS_DEFAULT_REGION").as_deref(), + Ok("us-west-2") + ); + assert_eq!( + std::env::var("AWS_ENDPOINT_URL_S3").as_deref(), + Ok("http://127.0.0.1:9000") + ); + + let ls = Cli::try_parse_from(["pchronicle", "-c", &config_arg, "ls", "@minio"])?; + unsafe { + std::env::remove_var("AWS_REGION"); + std::env::remove_var("AWS_DEFAULT_REGION"); + } + apply_catalog_backend_env_before_runtime(&ls)?; + assert_eq!(std::env::var("AWS_REGION").as_deref(), Ok("us-west-2")); + assert_eq!(std::env::var("AWS_ACCESS_KEY_ID").as_deref(), Ok("123")); + Ok(()) +} + #[tokio::test] async fn catalog_alias_stores_user_keys_and_rejects_endpoint() -> Result<()> { let temporary = tempfile::tempdir()?; @@ -657,6 +715,100 @@ async fn catalog_alias_stores_user_keys_and_rejects_endpoint() -> Result<()> { Ok(()) } +#[tokio::test] +async fn ls_catalog_alias_lists_authorized_datasets() -> Result<()> { + let temporary = tempfile::tempdir()?; + let catalog = temporary.path().join("catalog.toml"); + fs::write( + &catalog, + r#" +[datasets.prod] +uri = "s3://bucket/prod" +endpoint = "http://127.0.0.1:9000" +region = "us-west-2" +access_key = "BACKEND_AK" +secret_key = "BACKEND_SK" + +[datasets.evals] +uri = "s3://bucket/evals" +endpoint = "http://127.0.0.1:9000" +region = "us-west-2" +access_key = "BACKEND_AK" +secret_key = "BACKEND_SK" + +[users.alice] +access_key = "USER_AK" +secret_key = "USER_SK" + +[[grants]] +user = "alice" +dataset = "prod" +permissions = ["read"] + +[[grants]] +user = "alice" +dataset = "evals" +permissions = ["read"] +"#, + )?; + let acl = server::catalog::CatalogAcl::load(&catalog)?; + let app = server::PreparedWarehouse::prepare_catalog_front(acl) + .await? + .router(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { + axum::serve(listener, app).await.ok(); + }); + tokio::task::yield_now().await; + + let config = temporary.path().join("config.toml"); + let config_arg = config.to_string_lossy().into_owned(); + let cli = Cli::try_parse_from([ + "pchronicle", + "-c", + &config_arg, + "alias", + "add", + "team", + &format!("catalog://127.0.0.1:{port}"), + "--ak", + "USER_AK", + "--sk", + "USER_SK", + ])?; + run(cli, false, &mut Vec::new(), &mut Vec::new()).await?; + + for reference in ["@team", "@team/"] { + let ls = Cli::try_parse_from([ + "pchronicle", + "-c", + &config_arg, + "ls", + reference, + "--format", + "json", + ])?; + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + run(ls, false, &mut stdout, &mut stderr).await?; + let body = String::from_utf8(stdout)?; + assert!(body.contains("\"name\": \"prod\""), "{reference}: {body}"); + assert!(body.contains("\"name\": \"evals\""), "{reference}: {body}"); + assert!(body.contains("s3://bucket/prod"), "{reference}: {body}"); + assert!(!body.contains("BACKEND_AK"), "{reference}: {body}"); + assert!(!body.contains("BACKEND_SK"), "{reference}: {body}"); + assert!(!body.contains("USER_SK"), "{reference}: {body}"); + } + + // Keep non-ls resolution strict: bare catalog aliases still need a dataset. + let error = expand_dataset_reference("@team", Some(&config), false) + .unwrap_err() + .to_string(); + assert!(error.contains("requires a dataset"), "{error}"); + Ok(()) +} + #[tokio::test] async fn serve_catalog_issue_grant_revoke_rewrites_config() -> Result<()> { let temporary = tempfile::tempdir()?; @@ -761,6 +913,88 @@ secret_key = "BACKEND_SK" Ok(()) } +#[tokio::test] +async fn serve_catalog_dataset_add_list_remove_rewrites_config() -> Result<()> { + let temporary = tempfile::tempdir()?; + let root = temporary.path().join("data"); + fs::create_dir_all(&root)?; + let catalog = temporary.path().join("catalog.toml"); + fs::write( + &catalog, + format!( + r#" +[datasets.seed] +uri = "{}" +"#, + root.display() + ), + )?; + let catalog_arg = catalog.to_string_lossy().into_owned(); + let extra = temporary.path().join("extra"); + fs::create_dir_all(&extra)?; + let extra_uri = extra.to_string_lossy().into_owned(); + + let add = Cli::try_parse_from([ + "pchronicle", + "serve", + "catalog", + "dataset", + "add", + "--catalog-config", + &catalog_arg, + "extra", + "--uri", + &extra_uri, + "--format", + "json", + ])?; + let mut stdout = Vec::new(); + run(add, false, &mut stdout, &mut Vec::new()).await?; + let added: Value = serde_json::from_slice(&stdout)?; + assert_eq!(added["name"], "extra"); + + let list = Cli::try_parse_from([ + "pchronicle", + "serve", + "catalog", + "dataset", + "list", + "--catalog-config", + &catalog_arg, + "--format", + "json", + ])?; + let mut stdout = Vec::new(); + run(list, false, &mut stdout, &mut Vec::new()).await?; + let listed: Value = serde_json::from_slice(&stdout)?; + let names = listed + .as_array() + .unwrap() + .iter() + .map(|row| row["name"].as_str().unwrap().to_owned()) + .collect::>(); + assert!(names.contains(&"seed".to_string())); + assert!(names.contains(&"extra".to_string())); + + let remove = Cli::try_parse_from([ + "pchronicle", + "serve", + "catalog", + "dataset", + "remove", + "--catalog-config", + &catalog_arg, + "extra", + "--format", + "json", + ])?; + let mut stdout = Vec::new(); + run(remove, false, &mut stdout, &mut Vec::new()).await?; + let remaining: Value = serde_json::from_slice(&stdout)?; + assert_eq!(remaining["datasets"], serde_json::json!(["seed"])); + Ok(()) +} + #[test] fn alias_rejects_markdown_endpoint_links() { let error = super::s3_endpoint_for( @@ -876,6 +1110,8 @@ fn list_source_status_does_not_serialize_catalog_diagnostics() -> Result<()> { last_modified: None, status: CatalogSourceStatus::Error, error: Some("list-secret-sentinel /private/list/path".into()), + record_count: None, + failed_count: None, }; let output = serde_json::to_string(&source_response(&source))?; diff --git a/crates/persisting-pchronicle/Cargo.toml b/crates/persisting-pchronicle/Cargo.toml index c445472f..48bb277a 100644 --- a/crates/persisting-pchronicle/Cargo.toml +++ b/crates/persisting-pchronicle/Cargo.toml @@ -61,6 +61,7 @@ persisting-events.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value", "arbitrary_precision"] } serde_yaml.workspace = true +toml.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread", "sync"] } tracing.workspace = true diff --git a/crates/persisting-pchronicle/src/storage.rs b/crates/persisting-pchronicle/src/storage.rs index 2224caee..b524f303 100644 --- a/crates/persisting-pchronicle/src/storage.rs +++ b/crates/persisting-pchronicle/src/storage.rs @@ -31,22 +31,23 @@ pub use crate::store::{ CatalogErrorPolicy, CatalogEventProvenance, CatalogEventView, CatalogNamespace, CatalogPage, CatalogProjectionStatus, CatalogSnapshotOptions, CatalogSourceDescription, CatalogSourceKind, CatalogSourceRevision, CatalogSourceStatus, CatalogStorylineKey, CatalogTrajectoryBundle, - CommitRunOutcome, CompactJsonlColumn, CompactJsonlOffload, CompactJsonlOptions, - CompactJsonlRecord, CompactJsonlStore, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, + ChronicleManifest, CommitRunOutcome, CompactJsonlColumn, CompactJsonlOffload, + CompactJsonlOptions, CompactJsonlRecord, CompactJsonlStore, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, DEFAULT_DATASET_NAME, DEFAULT_MAX_EVENT_FALLBACK_BYTES, DEFAULT_MAX_EVENT_FALLBACK_ROWS, DEFAULT_PHYSICAL_PAGE_LIMIT, DatasetCatalogSnapshot, DatasetLocation, DatasetLocationKind, DatasetMount, DiscoveredSource, EventFactSnapshot, EventLogLayoutStats, EventWriterFence, ExportOutcome, LanceMaintenanceOptions, - LanceMaintenanceReport, LeaseAcquireOutcome, NamespacePath, ObjectStoreManifestWriteMode, - PhysicalColumn, PhysicalDataFile, PhysicalFileLayout, PhysicalFragment, PhysicalLayout, - PhysicalPage, PhysicalPagePreview, PhysicalPageQuery, PhysicalSource, PhysicalTable, - ProjectionSourceSnapshot, RawEventLanceAppender, RawEventLanceStore, ReplayOutcome, - RunControlStore, StorylineContentOptions, StorylineContentReadMode, StorylineDataSource, - StorylineDataSourceOptions, StorylineLanceStore, StorylineMaintenanceReport, - StorylineProjectionLineage, StorylineStreamImportReport, StorylineTablePaths, TrajectoryStats, - attempt_registry_now_ms, distinct_session_ids_in_run, export_source_dirs, export_story_bundle, - inspect_physical_file, inspect_physical_layout, inspect_physical_page, list_physical_sources, - raw_event_lance_path, + LanceMaintenanceReport, LeaseAcquireOutcome, ManifestKind, ManifestStats, NamespacePath, + ObjectStoreManifestWriteMode, PhysicalColumn, PhysicalDataFile, PhysicalFileLayout, + PhysicalFragment, PhysicalLayout, PhysicalPage, PhysicalPagePreview, PhysicalPageQuery, + PhysicalSource, PhysicalTable, ProjectionSourceSnapshot, RawEventLanceAppender, + RawEventLanceStore, ReplayOutcome, RunControlStore, StorylineContentOptions, + StorylineContentReadMode, StorylineDataSource, StorylineDataSourceOptions, StorylineLanceStore, + StorylineMaintenanceReport, StorylineProjectionLineage, StorylineStreamImportReport, + StorylineTablePaths, TrajectoryStats, attempt_registry_now_ms, distinct_session_ids_in_run, + export_source_dirs, export_story_bundle, inspect_physical_file, inspect_physical_layout, + inspect_physical_page, list_physical_sources, load_manifest, raw_event_lance_path, + write_compact_jsonl_manifest, }; // Compatibility exports; new callers should use `crate::search`. diff --git a/crates/persisting-pchronicle/src/store/catalog/discovery.rs b/crates/persisting-pchronicle/src/store/catalog/discovery.rs index 9e6a5be2..f7391e56 100644 --- a/crates/persisting-pchronicle/src/store/catalog/discovery.rs +++ b/crates/persisting-pchronicle/src/store/catalog/discovery.rs @@ -1,4 +1,5 @@ use super::*; +use crate::store::chronicle_manifest::{ManifestKind, try_load_manifest}; use crate::store::opendal_store::Store as OpendalStore; #[derive(Debug)] @@ -116,6 +117,8 @@ impl Candidate { last_modified, status: CatalogSourceStatus::Ready, error: None, + record_count: None, + failed_count: None, } } } @@ -171,15 +174,24 @@ pub(super) async fn freeze_candidate( )), )) } - Candidate::Compact { file, uri, .. } => Ok(( - source_row, - Arc::new(LazySource::new( - file, - LazySourceSpec::Compact { uri }, - options, - temporary_files, - )), - )), + Candidate::Compact { file, uri, .. } => { + if let Some(manifest) = + crate::store::chronicle_manifest::try_load_manifest(Path::new(&uri)) + && let Some(stats) = manifest.stats + { + source_row.record_count = Some(stats.record_count); + source_row.failed_count = Some(stats.failed_count); + } + Ok(( + source_row, + Arc::new(LazySource::new( + file, + LazySourceSpec::Compact { uri }, + options, + temporary_files, + )), + )) + } Candidate::LocalFile { file, root, @@ -540,6 +552,10 @@ async fn discover_local_candidates( "Dataset input is not a directory: {original_uri}" ); + if let Some(manifest) = try_load_manifest(root) { + return collect_manifest_subtree(root, root, &manifest, options).await; + } + if root.join("CURRENT").is_file() { let metadata = fs::metadata(root.join("CURRENT"))?; return Ok(vec![Candidate::Storyline { @@ -591,7 +607,10 @@ async fn discover_local_candidates( } let path = entry.path(); if file_type.is_dir() { - if path.join("CURRENT").is_file() { + if let Some(manifest) = try_load_manifest(&path) { + let nested = collect_manifest_subtree(root, &path, &manifest, options).await?; + candidates.extend(nested); + } else if path.join("CURRENT").is_file() { let metadata = fs::metadata(path.join("CURRENT"))?; candidates.push(Candidate::Storyline { file: relative_catalog_path(root, &path, true)?, @@ -646,16 +665,89 @@ async fn discover_local_candidates( Ok(candidates) } +async fn collect_manifest_subtree( + mount_root: &Path, + node: &Path, + manifest: &crate::store::ChronicleManifest, + options: LocalQueryManifestOptions, +) -> Result> { + let mut candidates = Vec::new(); + let mut stack = vec![(node.to_path_buf(), manifest.clone())]; + while let Some((current, current_manifest)) = stack.pop() { + match current_manifest.kind { + ManifestKind::Leaf => { + anyhow::ensure!( + current_manifest.is_compact_jsonl_leaf(), + "chronicle.manifest leaf format {:?} is not supported for discovery yet", + current_manifest.format + ); + let metadata = fs::metadata(¤t)?; + let file = if current == mount_root { + ".".into() + } else { + relative_catalog_path(mount_root, ¤t, true)? + }; + candidates.push(Candidate::Compact { + file, + uri: canonical_local_uri(¤t)?, + size_bytes: Some(metadata.len()), + last_modified: modified_string(&metadata), + }); + } + ManifestKind::Branch => { + let mut entries = fs::read_dir(¤t) + .with_context(|| { + format!("read chronicle.manifest branch {}", current.display()) + })? + .collect::>>()?; + entries.sort_by_key(|entry| entry.path()); + for entry in entries.into_iter().rev() { + anyhow::ensure!( + candidates.len() < options.max_files, + "Dataset manifest exceeds max_files limit of {}", + options.max_files + ); + let file_type = entry.file_type()?; + if file_type.is_symlink() || !file_type.is_dir() { + continue; + } + let child = entry.path(); + let Some(child_manifest) = try_load_manifest(&child) else { + continue; + }; + stack.push((child, child_manifest)); + } + } + } + anyhow::ensure!( + candidates.len() <= options.max_files, + "Dataset manifest exceeds max_files limit of {}", + options.max_files + ); + } + candidates.sort_by(|left, right| left.source_stub().file.cmp(&right.source_stub().file)); + Ok(candidates) +} + async fn is_compact_jsonl_directory(path: &Path) -> Result { + if let Some(manifest) = try_load_manifest(path) { + return Ok(manifest.is_compact_jsonl_leaf()); + } let dataset = match lance::Dataset::open(path.to_string_lossy().as_ref()).await { Ok(dataset) => dataset, Err(_) => return Ok(false), }; - Ok(dataset + let is_compact = dataset .schema() .metadata .get("pchronicle.format") - .is_some_and(|value| value == "compact-jsonl/v1")) + .is_some_and(|value| value == "compact-jsonl/v1"); + if is_compact { + // Store-layer upgrade path for pre-manifest datasets: first discovery + // that opens Lance also publishes chronicle.manifest. + let _ = crate::store::CompactJsonlStore::ensure_manifest(path).await?; + } + Ok(is_compact) } async fn discover_object_candidates( diff --git a/crates/persisting-pchronicle/src/store/catalog/mod.rs b/crates/persisting-pchronicle/src/store/catalog/mod.rs index 64ecd350..b9baa999 100644 --- a/crates/persisting-pchronicle/src/store/catalog/mod.rs +++ b/crates/persisting-pchronicle/src/store/catalog/mod.rs @@ -151,6 +151,11 @@ pub struct DiscoveredSource { pub last_modified: Option, pub status: CatalogSourceStatus, pub error: Option, + /// Aggregate record/run count from `chronicle.manifest` when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub record_count: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failed_count: Option, } impl DiscoveredSource { @@ -465,6 +470,29 @@ impl DatasetCatalogSnapshot { Ok(Some(crate::store::CompactJsonlStore::records(uri).await?)) } + /// Page compact-jsonl identities without materializing the full source. + pub async fn compact_records_page( + &self, + dataset: &str, + file: &str, + offset: usize, + limit: usize, + ) -> Result, u64)>> { + let key = CatalogStorylineKey { + dataset: dataset.into(), + file: file.into(), + document_id: String::new(), + session_id: String::new(), + }; + let source = self.lazy_source(&key)?; + let LazySourceSpec::Compact { uri } = &source.spec else { + return Ok(None); + }; + Ok(Some( + crate::store::CompactJsonlStore::records_page(uri, offset, limit).await?, + )) + } + pub async fn compact_record( &self, key: &CatalogStorylineKey, @@ -836,6 +864,7 @@ fn is_lance_directory(path: &Path) -> bool { path.extension() .and_then(|extension| extension.to_str()) .is_some_and(|extension| extension.eq_ignore_ascii_case("lance")) + || path.join("_versions").is_dir() } fn path_is_inside_lance_directory(path: &str) -> bool { diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index 89e027bb..462759e8 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -219,6 +219,169 @@ async fn ignores_derived_lance_sidecars_during_discovery() -> Result<()> { Ok(()) } +#[tokio::test] +async fn discovers_extensionless_compact_lance_dataset() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("input.jsonl"); + let compact = temp.path().join("compact"); + fs::write(&input, b"{\"timestamp\":1,\"value\":\"ok\"}\n")?; + crate::storage::CompactJsonlStore::import_path( + &input, + &compact, + &crate::storage::CompactJsonlOptions::default(), + ) + .await?; + fs::remove_file(input)?; + + let snapshot = DatasetCatalogSnapshot::discover( + vec![DatasetMount::default(temp.path().to_string_lossy())?], + Some(DEFAULT_DATASET_NAME.into()), + CatalogSnapshotOptions::default(), + ) + .await?; + let sources = &snapshot.datasets()[0].sources; + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].file, "compact"); + assert_eq!(sources[0].format.as_deref(), Some("compact-jsonl/v1")); + + let manifest = crate::storage::load_manifest(&compact)?.expect("import writes manifesto"); + assert!(manifest.is_compact_jsonl_leaf()); + assert_eq!(manifest.stats.as_ref().unwrap().record_count, 1); + Ok(()) +} + +#[tokio::test] +async fn discovers_nested_branch_and_leaf_chronicle_manifests_without_opening_lance() -> Result<()> +{ + let temp = tempfile::tempdir()?; + let warehouse = temp.path().join("warehouse"); + let leaf = warehouse.join("codex_jsonl"); + fs::create_dir_all(&leaf)?; + crate::store::chronicle_manifest::atomic_write_manifest( + &warehouse, + &crate::store::ChronicleManifest::branch(), + )?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&leaf, 1, 42)?; + // No Lance data/ tree: discovery must trust the leaf manifesto. + + let snapshot = DatasetCatalogSnapshot::discover( + vec![DatasetMount::default(warehouse.to_string_lossy())?], + Some(DEFAULT_DATASET_NAME.into()), + CatalogSnapshotOptions::default(), + ) + .await?; + let sources = &snapshot.datasets()[0].sources; + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].file, "codex_jsonl"); + assert_eq!(sources[0].format.as_deref(), Some("compact-jsonl/v1")); + assert_eq!(sources[0].record_count, Some(42)); + assert_eq!(sources[0].failed_count, Some(0)); + Ok(()) +} + +#[tokio::test] +async fn discovers_multi_level_branch_tree_and_preserves_leaf_counts() -> Result<()> { + let temp = tempfile::tempdir()?; + let warehouse = temp.path().join("warehouse"); + let team = warehouse.join("team"); + let leaf_a = team.join("codex_jsonl"); + let leaf_b = team.join("evals"); + let sibling = warehouse.join("archive"); + for dir in [&warehouse, &team, &leaf_a, &leaf_b, &sibling] { + fs::create_dir_all(dir)?; + } + crate::store::chronicle_manifest::atomic_write_manifest( + &warehouse, + &crate::store::ChronicleManifest::branch(), + )?; + crate::store::chronicle_manifest::atomic_write_manifest( + &team, + &crate::store::ChronicleManifest::branch(), + )?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&leaf_a, 1, 10)?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&leaf_b, 2, 20)?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&sibling, 3, 7)?; + + let snapshot = DatasetCatalogSnapshot::discover( + vec![DatasetMount::default(warehouse.to_string_lossy())?], + Some(DEFAULT_DATASET_NAME.into()), + CatalogSnapshotOptions::default(), + ) + .await?; + let sources = &snapshot.datasets()[0].sources; + let counts = sources + .iter() + .map(|source| (source.file.as_str(), source.record_count)) + .collect::>(); + assert_eq!( + counts, + vec![ + ("archive", Some(7)), + ("team/codex_jsonl", Some(10)), + ("team/evals", Some(20)), + ] + ); + // Branches are nesting only — never sources of their own. + assert!(sources.iter().all(|source| { + source.format.as_deref() == Some("compact-jsonl/v1") + && source.file != "team" + && source.file != "." + })); + Ok(()) +} + +#[tokio::test] +async fn leaf_manifest_does_not_recurse_into_nested_child_manifest() -> Result<()> { + let temp = tempfile::tempdir()?; + let leaf = temp.path().join("leaf"); + let nested = leaf.join("nested_child"); + fs::create_dir_all(&nested)?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&leaf, 1, 5)?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&nested, 1, 99)?; + + let snapshot = DatasetCatalogSnapshot::discover( + vec![DatasetMount::default(leaf.to_string_lossy())?], + Some(DEFAULT_DATASET_NAME.into()), + CatalogSnapshotOptions::default(), + ) + .await?; + let sources = &snapshot.datasets()[0].sources; + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].file, "."); + assert_eq!(sources[0].record_count, Some(5)); + Ok(()) +} + +#[tokio::test] +async fn updating_one_leaf_manifest_does_not_require_rewriting_parent_branch() -> Result<()> { + let temp = tempfile::tempdir()?; + let warehouse = temp.path().join("warehouse"); + let leaf = warehouse.join("codex_jsonl"); + fs::create_dir_all(&leaf)?; + crate::store::chronicle_manifest::atomic_write_manifest( + &warehouse, + &crate::store::ChronicleManifest::branch(), + )?; + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&leaf, 1, 10)?; + let parent_before = fs::read(warehouse.join("chronicle.manifest"))?; + + crate::store::chronicle_manifest::write_compact_jsonl_manifest(&leaf, 2, 42)?; + let parent_after = fs::read(warehouse.join("chronicle.manifest"))?; + assert_eq!( + parent_before, parent_after, + "leaf publish must not rewrite ancestor branch manifesto" + ); + + let snapshot = DatasetCatalogSnapshot::discover( + vec![DatasetMount::default(warehouse.to_string_lossy())?], + Some(DEFAULT_DATASET_NAME.into()), + CatalogSnapshotOptions::default(), + ) + .await?; + assert_eq!(snapshot.datasets()[0].sources[0].record_count, Some(42)); + Ok(()) +} + #[tokio::test] async fn report_mode_skips_oversized_files_when_querying_all_runs() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1064,6 +1227,8 @@ fn report_mode_source_status_does_not_serialize_operational_diagnostics() -> Res last_modified: None, status: CatalogSourceStatus::Ready, error: None, + record_count: None, + failed_count: None, }; let source = reported_source_failure( stub, @@ -1118,6 +1283,8 @@ mod proptests { last_modified: None, status: CatalogSourceStatus::Ready, error: None, + record_count: None, + failed_count: None, }, anyhow::anyhow!("secret diagnostic: {secret}"), ); diff --git a/crates/persisting-pchronicle/src/store/chronicle_manifest.rs b/crates/persisting-pchronicle/src/store/chronicle_manifest.rs new file mode 100644 index 00000000..c96c6aa6 --- /dev/null +++ b/crates/persisting-pchronicle/src/store/chronicle_manifest.rs @@ -0,0 +1,283 @@ +//! `chronicle.manifest` Dataset sidecar (RFC-0015). + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; + +pub const CHRONICLE_MANIFEST_FILE: &str = "chronicle.manifest"; +pub const CHRONICLE_MANIFEST_SCHEMA_VERSION: u32 = 1; +pub const COMPACT_JSONL_FORMAT: &str = "compact-jsonl/v1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManifestKind { + Leaf, + Branch, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManifestIdentity { + pub fingerprint: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManifestStats { + pub record_count: u64, + pub failed_count: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_timestamp: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_timestamp: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChronicleManifest { + pub schema_version: u32, + pub kind: ManifestKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identity: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stats: Option, +} + +impl ChronicleManifest { + pub fn leaf_compact_jsonl(fingerprint: impl Into, record_count: u64) -> Self { + Self { + schema_version: CHRONICLE_MANIFEST_SCHEMA_VERSION, + kind: ManifestKind::Leaf, + format: Some(COMPACT_JSONL_FORMAT.into()), + identity: Some(ManifestIdentity { + fingerprint: fingerprint.into(), + }), + stats: Some(ManifestStats { + record_count, + failed_count: 0, + min_timestamp: None, + max_timestamp: None, + total_tokens: None, + }), + } + } + + pub fn branch() -> Self { + Self { + schema_version: CHRONICLE_MANIFEST_SCHEMA_VERSION, + kind: ManifestKind::Branch, + format: None, + identity: None, + stats: None, + } + } + + pub fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == CHRONICLE_MANIFEST_SCHEMA_VERSION, + "unsupported chronicle.manifest schema_version {}; expected {}", + self.schema_version, + CHRONICLE_MANIFEST_SCHEMA_VERSION + ); + match self.kind { + ManifestKind::Leaf => { + let format = self + .format + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .context("leaf chronicle.manifest requires format")?; + ensure!( + !format.is_empty(), + "leaf chronicle.manifest format must not be empty" + ); + } + ManifestKind::Branch => { + ensure!( + self.format.is_none(), + "branch chronicle.manifest must not set format" + ); + } + } + if let Some(_stats) = &self.stats { + let identity = self + .identity + .as_ref() + .context("chronicle.manifest stats require [identity].fingerprint")?; + ensure!( + !identity.fingerprint.trim().is_empty(), + "chronicle.manifest fingerprint must not be empty" + ); + } + Ok(()) + } + + pub fn is_compact_jsonl_leaf(&self) -> bool { + self.kind == ManifestKind::Leaf + && self.format.as_deref() == Some(COMPACT_JSONL_FORMAT) + && self.validate().is_ok() + } +} + +pub fn manifest_path(root: impl AsRef) -> PathBuf { + root.as_ref().join(CHRONICLE_MANIFEST_FILE) +} + +pub fn lance_version_fingerprint(version: u64) -> String { + format!("lance:version:{version}") +} + +pub fn load_manifest(root: impl AsRef) -> Result> { + let path = manifest_path(root); + if !path.is_file() { + return Ok(None); + } + let text = fs::read_to_string(&path) + .with_context(|| format!("read chronicle.manifest {}", path.display()))?; + let manifest: ChronicleManifest = toml::from_str(&text) + .with_context(|| format!("parse chronicle.manifest {}", path.display()))?; + manifest.validate()?; + Ok(Some(manifest)) +} + +pub fn try_load_manifest(root: impl AsRef) -> Option { + match load_manifest(root) { + Ok(manifest) => manifest, + Err(error) => { + tracing::warn!( + target: "persisting_pchronicle::chronicle_manifest", + error = %error, + "ignoring invalid chronicle.manifest" + ); + None + } + } +} + +pub fn atomic_write_manifest(root: impl AsRef, manifest: &ChronicleManifest) -> Result<()> { + manifest.validate()?; + let root = root.as_ref(); + fs::create_dir_all(root) + .with_context(|| format!("create chronicle.manifest parent {}", root.display()))?; + let path = manifest_path(root); + let temporary = root.join(format!( + ".{}.tmp-{}", + CHRONICLE_MANIFEST_FILE, + std::process::id() + )); + let encoded = toml::to_string_pretty(manifest).context("encode chronicle.manifest")?; + { + let mut file = fs::File::create(&temporary) + .with_context(|| format!("create chronicle.manifest temp {}", temporary.display()))?; + file.write_all(encoded.as_bytes()) + .with_context(|| format!("write chronicle.manifest temp {}", temporary.display()))?; + file.sync_all() + .with_context(|| format!("sync chronicle.manifest temp {}", temporary.display()))?; + } + fs::rename(&temporary, &path).with_context(|| { + let _ = fs::remove_file(&temporary); + format!( + "publish chronicle.manifest {} from {}", + path.display(), + temporary.display() + ) + })?; + Ok(()) +} + +pub fn write_compact_jsonl_manifest( + root: impl AsRef, + lance_version: u64, + record_count: u64, +) -> Result<()> { + let manifest = ChronicleManifest::leaf_compact_jsonl( + lance_version_fingerprint(lance_version), + record_count, + ); + atomic_write_manifest(root, &manifest) +} + +/// True when a compact-jsonl leaf manifesto matches one Lance version. +pub fn compact_jsonl_manifest_matches(manifest: &ChronicleManifest, lance_version: u64) -> bool { + manifest.is_compact_jsonl_leaf() + && manifest.identity.as_ref().is_some_and(|identity| { + identity.fingerprint == lance_version_fingerprint(lance_version) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn leaf_round_trip_and_validation() { + let manifest = ChronicleManifest::leaf_compact_jsonl("lance:version:3", 12); + manifest.validate().unwrap(); + assert!(manifest.is_compact_jsonl_leaf()); + let encoded = toml::to_string_pretty(&manifest).unwrap(); + let decoded: ChronicleManifest = toml::from_str(&encoded).unwrap(); + assert_eq!(decoded, manifest); + } + + #[test] + fn branch_rejects_format() { + let mut manifest = ChronicleManifest::branch(); + manifest.format = Some(COMPACT_JSONL_FORMAT.into()); + assert!(manifest.validate().is_err()); + } + + #[test] + fn atomic_write_creates_readable_file() { + let dir = tempdir().unwrap(); + let manifest = ChronicleManifest::leaf_compact_jsonl("lance:version:1", 4); + atomic_write_manifest(dir.path(), &manifest).unwrap(); + let loaded = load_manifest(dir.path()).unwrap().unwrap(); + assert_eq!(loaded.stats.unwrap().record_count, 4); + assert_eq!(loaded.identity.unwrap().fingerprint, "lance:version:1"); + } + + #[test] + fn load_missing_returns_none() { + let dir = tempdir().unwrap(); + assert!(load_manifest(dir.path()).unwrap().is_none()); + } + + #[test] + fn reject_unsupported_schema_version() { + let err = ChronicleManifest { + schema_version: 99, + kind: ManifestKind::Branch, + format: None, + identity: None, + stats: None, + } + .validate() + .unwrap_err(); + assert!(err.to_string().contains("schema_version")); + } + + #[test] + fn stats_require_fingerprint() { + let err = ChronicleManifest { + schema_version: 1, + kind: ManifestKind::Leaf, + format: Some(COMPACT_JSONL_FORMAT.into()), + identity: None, + stats: Some(ManifestStats { + record_count: 1, + failed_count: 0, + min_timestamp: None, + max_timestamp: None, + total_tokens: None, + }), + } + .validate() + .unwrap_err(); + assert!(err.to_string().contains("identity")); + } +} diff --git a/crates/persisting-pchronicle/src/store/compact_jsonl.rs b/crates/persisting-pchronicle/src/store/compact_jsonl.rs index ae112c6c..787f5691 100644 --- a/crates/persisting-pchronicle/src/store/compact_jsonl.rs +++ b/crates/persisting-pchronicle/src/store/compact_jsonl.rs @@ -19,6 +19,8 @@ use lance_arrow::json::{decode_json, encode_json, json_field}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use crate::store::ChronicleManifest; + const RAW_COLUMN: &str = "_raw_"; const OFFLOAD_COLUMN: &str = "_offload_"; const FORMAT_KEY: &str = "pchronicle.format"; @@ -113,8 +115,130 @@ pub struct CompactJsonlRecord { pub struct CompactJsonlStore; impl CompactJsonlStore { + /// Store-layer publish: every successful compact-jsonl Lance write MUST end + /// here so catalog/CLI paths cannot skip `chronicle.manifest`. + pub async fn publish_manifest(root: impl AsRef) -> Result { + let root = root.as_ref(); + let dataset = Dataset::open(root.to_string_lossy().as_ref()) + .await + .with_context(|| { + format!( + "open compact JSONL for chronicle.manifest {}", + root.display() + ) + })?; + validate_dataset_schema(&dataset)?; + let version = dataset.version_id(); + let record_count = dataset.count_rows(None).await? as u64; + crate::store::chronicle_manifest::write_compact_jsonl_manifest( + root, + version, + record_count, + )?; + crate::store::chronicle_manifest::load_manifest(root)? + .context("chronicle.manifest missing after publish") + } + + /// Ensure the sidecar matches the opened Lance revision. Missing or stale + /// manifests are rewritten; compatible old datasets are upgraded in place. + pub async fn ensure_manifest(root: impl AsRef) -> Result> { + let root = root.as_ref(); + let dataset = match Dataset::open(root.to_string_lossy().as_ref()).await { + Ok(dataset) => dataset, + Err(_) => return Ok(None), + }; + if validate_dataset_schema(&dataset).is_err() { + return Ok(None); + } + let version = dataset.version_id(); + if let Some(manifest) = crate::store::chronicle_manifest::try_load_manifest(root) + && crate::store::chronicle_manifest::compact_jsonl_manifest_matches(&manifest, version) + { + return Ok(Some(manifest)); + } + let record_count = dataset.count_rows(None).await? as u64; + match crate::store::chronicle_manifest::write_compact_jsonl_manifest( + root, + version, + record_count, + ) { + Ok(()) => Ok(crate::store::chronicle_manifest::try_load_manifest(root)), + Err(error) => { + tracing::warn!( + target: "persisting_pchronicle::compact_jsonl", + error = %error, + path = %root.display(), + "failed to ensure chronicle.manifest for compact JSONL dataset" + ); + Ok(None) + } + } + } + + /// Page compact record identities without loading the full table into memory. + pub async fn records_page( + input: impl AsRef, + offset: usize, + limit: usize, + ) -> Result<(Vec, u64)> { + let input = input.as_ref(); + let limit = limit.max(1); + let manifest = Self::ensure_manifest(input).await?; + let dataset = Dataset::open(input.to_string_lossy().as_ref()).await?; + validate_dataset_schema(&dataset)?; + let total = if let Some(count) = manifest + .as_ref() + .and_then(|manifest| manifest.stats.as_ref()) + .map(|stats| stats.record_count) + { + count + } else { + dataset.count_rows(None).await? as u64 + }; + if offset as u64 >= total || limit == 0 { + return Ok((Vec::new(), total)); + } + let mut scan = dataset.scan(); + scan.scan_in_order(true); + scan.limit(Some(limit as i64), (offset > 0).then_some(offset as i64)) + .context("apply compact JSONL page offset/limit")?; + let stream = scan.try_into_stream().await?; + let mut stream = stream; + let mut out = Vec::with_capacity(limit.min(4096)); + while let Some(batch) = stream.try_next().await? { + let ids = batch + .column(batch.schema().index_of("id")?) + .as_any() + .downcast_ref::() + .context("compact JSONL id must be Utf8")?; + let timestamps = batch + .column(batch.schema().index_of("timestamp")?) + .as_any() + .downcast_ref::() + .context("compact JSONL timestamp must be Utf8")?; + let filenames = batch + .column(batch.schema().index_of("filename")?) + .as_any() + .downcast_ref::() + .context("compact JSONL filename must be Utf8")?; + for row in 0..batch.num_rows() { + if out.len() >= limit { + return Ok((out, total)); + } + out.push(CompactJsonlRecord { + id: ids.value(row).into(), + timestamp: timestamps.value(row).into(), + filename: filenames.value(row).into(), + }); + } + } + Ok((out, total)) + } + pub async fn records(input: impl AsRef) -> Result> { - let dataset = Dataset::open(input.as_ref().to_string_lossy().as_ref()).await?; + let input = input.as_ref(); + let _ = Self::ensure_manifest(input).await?; + let dataset = Dataset::open(input.to_string_lossy().as_ref()).await?; validate_dataset_schema(&dataset)?; let stream = dataset.scan().scan_in_order(true).try_into_stream().await?; let mut stream = stream; @@ -149,7 +273,9 @@ impl CompactJsonlStore { /// Read one record for the Web explorer without assigning trajectory /// semantics to the compact row. pub async fn read_record(input: impl AsRef, id: &str) -> Result> { - let dataset = Dataset::open(input.as_ref().to_string_lossy().as_ref()).await?; + let input = input.as_ref(); + let _ = Self::ensure_manifest(input).await?; + let dataset = Dataset::open(input.to_string_lossy().as_ref()).await?; let (_, offload_idx) = validate_dataset_schema(&dataset)?; let stream = dataset.scan().scan_in_order(true).try_into_stream().await?; let mut stream = stream; @@ -173,8 +299,7 @@ impl CompactJsonlStore { let descriptor = json_text_at(offloads.as_ref(), row)? .context("compact JSONL row has neither data nor offload")?; let reference: CompactJsonlOffload = serde_json::from_str(&descriptor)?; - let bytes = - fs::read(input.as_ref().join(&reference.path).join(&reference.key))?; + let bytes = fs::read(input.join(&reference.path).join(&reference.key))?; ensure!( blake3::hash(&bytes).to_hex().as_str() == reference.key, "compact JSONL offload digest mismatch" @@ -195,10 +320,10 @@ impl CompactJsonlStore { let input = input.as_ref(); let output = output.as_ref(); options.validate()?; - let files = collect_jsonl(input)?; + let files = collect_json_inputs(input)?; ensure!( !files.is_empty(), - "compact JSONL input contains no .jsonl files" + "compact JSONL input contains no .json, .jsonl, or .ndjson files" ); if output.exists() { fs::remove_dir_all(output) @@ -216,49 +341,63 @@ impl CompactJsonlStore { .to_str() .context("compact JSONL filename is not UTF-8")? .replace('\\', "/"); - let mut reader = BufReader::new(File::open(&file)?); let first_row = rows.len(); - for line_no in 1usize.. { - let mut raw = Vec::new(); - if reader.read_until(b'\n', &mut raw)? == 0 { - break; - } - let mut end = raw.len() - usize::from(raw.ends_with(b"\n")); - end -= usize::from(raw[..end].ends_with(b"\r")); - let json = &raw[..end]; - ensure!( - !json.iter().all(u8::is_ascii_whitespace), - "compact JSONL {}:{} is empty", - relative, - line_no - ); - let value: Value = serde_json::from_slice(json) - .with_context(|| format!("parse compact JSONL {relative}:{line_no}"))?; + if is_json_document(&file) { + let raw = fs::read(&file)?; + let value: Value = serde_json::from_slice(&raw) + .with_context(|| format!("parse compact JSON {relative}"))?; ensure!( - value.is_object(), - "compact JSONL {relative}:{line_no} must be a JSON object" + matches!(value, Value::Object(_) | Value::Array(_)), + "compact JSON {relative} must be an object or array" ); - rows.push((value, raw, relative.clone(), line_no)); + rows.push((value, raw, relative.clone(), 1)); + } else { + let mut reader = BufReader::new(File::open(&file)?); + for line_no in 1usize.. { + let mut raw = Vec::new(); + if reader.read_until(b'\n', &mut raw)? == 0 { + break; + } + let mut end = raw.len() - usize::from(raw.ends_with(b"\n")); + end -= usize::from(raw[..end].ends_with(b"\r")); + let json = &raw[..end]; + ensure!( + !json.iter().all(u8::is_ascii_whitespace), + "compact JSONL {}:{} is empty", + relative, + line_no + ); + let value: Value = serde_json::from_slice(json) + .with_context(|| format!("parse compact JSONL {relative}:{line_no}"))?; + ensure!( + value.is_object(), + "compact JSONL {relative}:{line_no} must be a JSON object" + ); + rows.push((value, raw, relative.clone(), line_no)); + } } ensure!(rows.len() > first_row, "compact JSONL {relative} is empty"); } let schema = schema(options)?; let mut arrays: Vec> = Vec::new(); - let required = |name: &str, default| -> Result> { - let path = options.path_for(name, default); - rows.iter() - .map(|(value, _, file, line)| { - path_value(value, path) + let (ids, timestamps): (Vec<_>, Vec<_>) = rows + .iter() + .map(|(value, _, file, line)| { + let generated = generated_record_key(file, *line); + let timestamp = path_value(value, options.path_for("timestamp", "$.timestamp")) + .and_then(scalar_string) + .filter(|value| !value.is_empty()); + let id = if timestamp.is_none() { + generated.clone() + } else { + path_value(value, options.path_for("id", "$.id")) .and_then(scalar_string) .filter(|value| !value.is_empty()) - .with_context(|| { - format!("compact JSONL {file}:{line} requires scalar {name} at {path}") - }) - }) - .collect() - }; - let ids = required("id", "$.id")?; - let timestamps = required("timestamp", "$.timestamp")?; + .unwrap_or_else(|| generated.clone()) + }; + (id, timestamp.unwrap_or(generated)) + }) + .unzip(); let mut unique_ids = std::collections::HashSet::new(); for id in &ids { ensure!(unique_ids.insert(id), "duplicate compact JSONL id '{id}'"); @@ -359,6 +498,9 @@ impl CompactJsonlStore { .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema)) .await .context("write compact JSONL Lance dataset")?; + // Store-layer contract: every published compact dataset carries + // chronicle.manifest. import and sync both end here. + Self::publish_manifest(output).await?; Ok(rows.len()) } @@ -530,7 +672,7 @@ fn encode_json_bytes(value: &str) -> Result> { encode_json(value).map_err(|error| anyhow!("encode compact JSON value: {error}")) } -fn collect_jsonl(input: &Path) -> Result> { +fn collect_json_inputs(input: &Path) -> Result> { let metadata = fs::symlink_metadata(input) .with_context(|| format!("inspect compact JSONL input {}", input.display()))?; ensure!( @@ -542,8 +684,11 @@ fn collect_jsonl(input: &Path) -> Result> { input .extension() .and_then(|x| x.to_str()) - .is_some_and(|x| x.eq_ignore_ascii_case("jsonl")), - "compact JSONL input must be .jsonl" + .is_some_and(|x| matches!( + x.to_ascii_lowercase().as_str(), + "json" | "jsonl" | "ndjson" + )), + "compact JSONL input must be .json, .jsonl, or .ndjson" ); return Ok(vec![input.to_path_buf()]); } @@ -563,11 +708,7 @@ fn collect_jsonl(input: &Path) -> Result> { let p = entry.path(); if file_type.is_dir() { stack.push(p); - } else if file_type.is_file() - && p.extension() - .and_then(|x| x.to_str()) - .is_some_and(|x| x.eq_ignore_ascii_case("jsonl")) - { + } else if file_type.is_file() && is_json_input(&p) { out.push(p); } } @@ -576,6 +717,27 @@ fn collect_jsonl(input: &Path) -> Result> { Ok(out) } +fn is_json_input(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + matches!( + extension.to_ascii_lowercase().as_str(), + "json" | "jsonl" | "ndjson" + ) + }) +} + +fn is_json_document(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("json")) +} + +fn generated_record_key(file: &str, line: usize) -> String { + format!("{file}#{line}") +} + fn valid_path(path: &str) -> bool { if path == "$" { return true; @@ -690,6 +852,51 @@ mod tests { } #[tokio::test] + async fn ensure_manifest_backfills_missing_sidecar_for_legacy_datasets() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("input.jsonl"); + let dataset = temp.path().join("data.lance"); + fs::write( + &input, + b"{\"id\":\"a\",\"timestamp\":1}\n{\"id\":\"b\",\"timestamp\":2}\n", + )?; + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()).await?; + fs::remove_file(crate::store::chronicle_manifest::manifest_path(&dataset))?; + assert!(crate::store::load_manifest(&dataset)?.is_none()); + + let ensured = CompactJsonlStore::ensure_manifest(&dataset) + .await? + .expect("ensure rewrites manifesto"); + assert!(ensured.is_compact_jsonl_leaf()); + assert_eq!(ensured.stats.as_ref().unwrap().record_count, 2); + assert!(crate::store::load_manifest(&dataset)?.is_some()); + Ok(()) + } + + #[tokio::test] + async fn records_page_returns_offset_window_and_manifest_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("input.jsonl"); + let dataset = temp.path().join("data.lance"); + let mut body = String::new(); + for index in 0..4 { + body.push_str(&format!( + "{{\"id\":\"id-{index}\",\"timestamp\":{index}}}\n" + )); + } + fs::write(&input, body)?; + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()).await?; + + let (page, total) = CompactJsonlStore::records_page(&dataset, 1, 2).await?; + assert_eq!(total, 4); + assert_eq!(page.len(), 2); + assert_eq!(page[0].id, "id-1"); + assert_eq!(page[1].id, "id-2"); + Ok(()) + } + + #[tokio::test] + async fn required_columns_can_be_remapped_and_final_newline_is_exact() -> Result<()> { let temp = tempfile::tempdir()?; let input = temp.path().join("input.jsonl"); @@ -711,18 +918,70 @@ mod tests { } #[tokio::test] - async fn missing_required_timestamp_rejects_the_snapshot() -> Result<()> { + async fn missing_base_columns_use_source_line_values() -> Result<()> { let temp = tempfile::tempdir()?; let input = temp.path().join("input.jsonl"); fs::write(&input, b"{\"id\":\"only\"}\n")?; - let error = CompactJsonlStore::import_path( - &input, - temp.path().join("data.lance"), - &CompactJsonlOptions::default(), - ) - .await - .unwrap_err(); - assert!(error.to_string().contains("requires scalar timestamp")); + let dataset = temp.path().join("data.lance"); + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()).await?; + let record = &CompactJsonlStore::records(&dataset).await?[0]; + assert_eq!(record.id, "input.jsonl#1"); + assert_eq!(record.timestamp, "input.jsonl#1"); + Ok(()) + } + + #[tokio::test] + async fn json_documents_and_arrays_are_imported_as_records() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("input"); + let dataset = temp.path().join("data.lance"); + let output = temp.path().join("out"); + fs::create_dir_all(&input)?; + fs::write(input.join("object.json"), b"{\"value\":1}")?; + fs::write(input.join("array.json"), b"[{\"value\":2},{\"value\":3}]")?; + + assert_eq!( + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()) + .await?, + 2 + ); + let records = CompactJsonlStore::records(&dataset).await?; + assert_eq!( + records + .iter() + .map(|record| record.id.as_str()) + .collect::>(), + ["array.json#1", "object.json#1"] + ); + CompactJsonlStore::export_path(&dataset, &output).await?; + assert_eq!( + fs::read(output.join("array.json"))?, + b"[{\"value\":2},{\"value\":3}]" + ); + assert_eq!(fs::read(output.join("object.json"))?, b"{\"value\":1}"); + Ok(()) + } + + #[tokio::test] + async fn missing_id_uses_source_line_and_preserves_input() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("events.jsonl"); + let dataset = temp.path().join("data.lance"); + let output = temp.path().join("out"); + let raw = b"{\"timestamp\":1,\"event\":\"start\"}\n{\"timestamp\":2,\"event\":\"end\"}\n"; + fs::write(&input, raw)?; + + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()).await?; + let records = CompactJsonlStore::records(&dataset).await?; + assert_eq!( + records + .iter() + .map(|record| record.id.as_str()) + .collect::>(), + ["events.jsonl#1", "events.jsonl#2"] + ); + CompactJsonlStore::export_path(&dataset, &output).await?; + assert_eq!(fs::read(output.join("events.jsonl"))?, raw); Ok(()) } } diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 9b803ca3..76ee0980 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -16,6 +16,8 @@ mod cas_store; #[cfg(feature = "lance-store")] mod catalog; #[cfg(feature = "lance-store")] +mod chronicle_manifest; +#[cfg(feature = "lance-store")] mod compact_jsonl; #[cfg(feature = "lance-store")] mod datafusion_bridge; @@ -71,6 +73,12 @@ pub use catalog::{ DatasetCatalogSnapshot, DatasetMount, DiscoveredSource, NamespacePath, }; #[cfg(feature = "lance-store")] +#[allow(unused_imports)] +pub use chronicle_manifest::{ + CHRONICLE_MANIFEST_FILE, ChronicleManifest, ManifestKind, ManifestStats, atomic_write_manifest, + compact_jsonl_manifest_matches, load_manifest, try_load_manifest, write_compact_jsonl_manifest, +}; +#[cfg(feature = "lance-store")] pub use compact_jsonl::{ CompactJsonlColumn, CompactJsonlOffload, CompactJsonlOptions, CompactJsonlRecord, CompactJsonlStore, diff --git a/crates/persisting-pchronicle/tests/proptests/catalog_provenance.rs b/crates/persisting-pchronicle/tests/proptests/catalog_provenance.rs index 03f48abd..bddb4c1c 100644 --- a/crates/persisting-pchronicle/tests/proptests/catalog_provenance.rs +++ b/crates/persisting-pchronicle/tests/proptests/catalog_provenance.rs @@ -64,6 +64,8 @@ proptest! { CatalogSourceStatus::Error }, error: None, + record_count: None, + failed_count: None, }) .collect(); let dataset = CatalogDataset { diff --git a/docs/sidebars.js b/docs/sidebars.js index c6d7bdf6..4e4d881b 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -14,6 +14,7 @@ module.exports = { { type: 'category', label: 'Guides', items: [{ type: 'autogenerated', dirName: 'guides' }] }, { type: 'category', label: 'pVisor', items: product('pvisor', { overview: 'Overview', getStarted: 'Get Started', concepts: 'Concepts', guides: 'Guides', design: 'Design', reference: 'Reference' }) }, { type: 'category', label: 'pChronicle', items: product('pchronicle', { overview: 'Overview', getStarted: 'Get Started', concepts: 'Concepts', guides: 'Guides', design: 'Design', reference: 'Reference' }) }, + { type: 'category', label: 'pPilot (preview)', items: [{ type: 'autogenerated', dirName: 'ppilot' }] }, { type: 'category', label: 'System design', items: [{ type: 'autogenerated', dirName: 'system-design' }] }, { type: 'category', label: 'Project', items: [{ type: 'autogenerated', dirName: 'project' }] }, { type: 'category', label: 'RFCs', items: [{ type: 'autogenerated', dirName: 'rfcs' }] }, @@ -25,6 +26,7 @@ module.exports = { { type: 'category', label: '指南', items: [{ type: 'autogenerated', dirName: 'guides' }] }, { type: 'category', label: 'pVisor', items: product('pvisor', { overview: '概览', getStarted: '快速开始', concepts: '核心概念', guides: '指南', design: '设计', reference: '参考' }) }, { type: 'category', label: 'pChronicle', items: product('pchronicle', { overview: '概览', getStarted: '快速开始', concepts: '核心概念', guides: '指南', design: '设计', reference: '参考' }) }, + { type: 'category', label: 'pPilot(预览)', items: [{ type: 'autogenerated', dirName: 'ppilot' }] }, { type: 'category', label: '系统设计', items: [{ type: 'autogenerated', dirName: 'system-design' }] }, { type: 'category', label: '项目', items: [{ type: 'autogenerated', dirName: 'project' }] }, { type: 'category', label: 'RFC', items: [{ type: 'autogenerated', dirName: 'rfcs' }] }, diff --git a/docs/src/en/index.md b/docs/src/en/index.md index 4b9e3615..3ef5dbc5 100644 --- a/docs/src/en/index.md +++ b/docs/src/en/index.md @@ -7,22 +7,18 @@ sidebar_label: Start here Persisting gives you two independent product paths. Choose the one that matches the work in front of you: -- [Run an Agent safely with pVisor](pvisor/get-started.md): stage workspace changes, inspect the Run Bundle, and apply only what you approve. -- [Explore durable history with pChronicle](pchronicle/get-started.md): open a Dataset, run a read-only query, and follow Source lineage. -- [Understand the product boundary](overview.md): see how execution and history connect without becoming one opaque system. +- [Run an Agent safely with pVisor](pvisor/get-started.md): run the Agent in a staged workspace, inspect its changes, and write only what you approve into the project. +- [Explore durable history with pChronicle](pchronicle/get-started.md): open trajectory data, run a read-only query, and know which data and which source you are reading. +- [Choose a workflow](overview.md): decide which path to take, and how execution and history can optionally connect. -If you are evaluating the system, start with [Choose a workflow](overview.md), then follow the product walkthrough that matches your data. +If you are evaluating the system, start with [Choose a workflow](overview.md), then follow the matching product walkthrough. ## What you will have after the first walkthrough -- A **pVisor** walkthrough ends with a stopped Run, a readable Run Bundle, and - a deliberate apply or drop decision. Your project is changed only when you - choose to apply the staged Effect. -- A **pChronicle** walkthrough ends with a read-only Dataset query and a clear - distinction between the Dataset, its Source, and the Snapshot being read. +- **pVisor**: the Agent has stopped; its changes remain in a staging directory; you deliberately write them into the project or discard them. The project changes only when you choose to write. +- **pChronicle**: you have run a read-only query against trajectory data and know which data and which source you inspected. -You do not need both products to begin. Add the capture handoff only when you -need to correlate execution evidence with durable trajectory history. +You do not need both products to begin. Add the capture handoff only when you need to correlate one execution with durable trajectory history. ## Before you start diff --git a/docs/src/en/overview.md b/docs/src/en/overview.md index 1db61bc8..1c74b21e 100644 --- a/docs/src/en/overview.md +++ b/docs/src/en/overview.md @@ -5,17 +5,17 @@ the work in front of you, then follow the short path to a useful result. ## I want to run an Agent safely and review its changes -Start with **pVisor**. It gives one Agent a Run-owned workspace, records the -execution boundary, and leaves filesystem changes staged until you decide what -enters the real project. +Start with **pVisor**. It lets one Agent work in an isolated Run, records the execution boundary, and +leaves filesystem changes staged until you decide to write them into the real +project or discard them. 1. [Install the command line tools](installation.md). 2. [Run your first Agent](pvisor/get-started.md). 3. [Review and selectively apply changes](pvisor/guides/review-apply.md). 4. [Choose a host, OCI, or VM environment](pvisor/guides/execution.md). -You should finish with a completed Run, a readable Run Bundle, and either an -applied or discarded stage. +When it finishes, the Agent has stopped and its changes remain staged for you to +review, apply, or discard. ```bash pvisor run --stage ./runs/task-001 -- codex @@ -34,8 +34,8 @@ example data, so you can learn the query flow before preparing a Dataset. 3. [Import or export a supported format](pchronicle/guides/exchange.md). 4. [Serve a Dataset locally](pchronicle/guides/serve.md). -You should finish with a read-only query, a normalized view, and a clear Source -lineage for the data you inspected. +You should finish with a read-only query, a normalized view, and a clear sense +of which data and which source you inspected. ```bash pchronicle onboard query @@ -51,8 +51,13 @@ records into pChronicle. The handoff is explicit and narrow: it does not move the private Run Bundle or invent evidence that the original Source did not provide. +pVisor capture and `pchronicle serve --gateway` are separate entry points: pVisor capture +shares an Agent Run lifecycle and boundary; `pchronicle serve --gateway` receives or forwards +traffic from an existing Agent or SDK without starting a pVisor Run. + 1. [Capture Agent trajectories](pvisor/guides/capture.md). -2. [Understand the event and sidecar contract](rfcs/0007-events-contract-pchronicle-sidecar.md). +2. [Understand the event and sidecar contract](rfcs/0007-events-contract-pchronicle-sidecar.md) + (for protocol changes or troubleshooting). 3. [Read the execution-to-history architecture](system-design/architecture.md). ```text diff --git a/docs/src/en/pchronicle/index.md b/docs/src/en/pchronicle/index.md index d4bc2e08..6f0f0754 100644 --- a/docs/src/en/pchronicle/index.md +++ b/docs/src/en/pchronicle/index.md @@ -6,14 +6,13 @@ exchange, and serve run Datasets produced by Persisting or by supported external formats; pChronicle does not require pVisor to run. -Within Persisting's model-state-to-Agent-history story, pChronicle is the -durable store and query engine for Agent history. It can run as a local tool -or be deployed as a platform in front of many paths. +In Persisting, pChronicle stores and queries trajectory history. It does not +require pVisor. It can run as a local tool or be deployed as a service in front +of many paths. :::tip What you will complete -The first walkthrough creates temporary data, opens it as a Dataset, runs a -read-only summary, and answers one SQL question. You can learn the query model -without preparing a production store first. +The first walkthrough creates temporary data, opens it, runs a read-only +summary, and answers one SQL question. You do not need a production store first. ::: ## The one object you work with diff --git a/docs/src/en/pchronicle/reference/cli.md b/docs/src/en/pchronicle/reference/cli.md index c1005932..df5b3d73 100644 --- a/docs/src/en/pchronicle/reference/cli.md +++ b/docs/src/en/pchronicle/reference/cli.md @@ -210,9 +210,11 @@ replaced in place. Compact JSONL is a record store, not a trajectory conversion. Either `--input-format compact-jsonl` or `--output-format compact-jsonl` selects it. -It recursively reads local `.jsonl` files and requires every non-empty line to -be a JSON object with a unique scalar `id` and scalar `timestamp`; their default -paths are `$.id` and `$.timestamp`. `--column id=PATH` and +It recursively reads local `.json`, `.jsonl`, and `.ndjson` files. JSON objects +and arrays are accepted; each JSON document becomes one record, with arrays kept intact. +Missing or invalid `id` and `timestamp` values receive stable +`source_filename#line_number` values. The default paths are `$.id` and +`$.timestamp`. `--column id=PATH` and `--column timestamp=PATH` override those paths, while any other `--column NAME=PATH` adds a nullable JSONB projection. Compact import supports local `create` and confirmed `replace`, but not stdin, object-store targets, or @@ -235,7 +237,8 @@ the set and retry with bounded exponential backoff. Use `--once` for one initial batch and exit. The two destinations must be local directories outside the source directory. -With `--input-format compact-jsonl`, the source must be a local `.jsonl` tree +With `--input-format compact-jsonl`, the source must be a local `.json`, `.jsonl`, +or `.ndjson` tree and the same `--column` rules as compact import apply. Every successful batch rescans the whole tree and atomically replaces the compact Lance snapshot at `--convert`, so additions, changes, and deletions are reflected without diff --git a/docs/src/en/project/index.md b/docs/src/en/project/index.md index bd45bb17..f89363a3 100644 --- a/docs/src/en/project/index.md +++ b/docs/src/en/project/index.md @@ -1,8 +1,8 @@ # Project -Persisting's public positioning spans model state and Agent history. This +Persisting's public product path is pVisor and pChronicle. This section records delivery state, durable decisions, contributor workflows, and -systems outside the current pVisor and pChronicle product path. +systems outside that current path. ## Architecture diff --git a/docs/src/en/pvisor/concepts/agentvisor.md b/docs/src/en/pvisor/concepts/agentvisor.md index 192a8cf5..bd2c9552 100644 --- a/docs/src/en/pvisor/concepts/agentvisor.md +++ b/docs/src/en/pvisor/concepts/agentvisor.md @@ -1,5 +1,11 @@ # What is an AgentVisor? +:::note How to read this page +This page defines the **AgentVisor category**. It is not a pVisor feature list or +delivery commitment. What you can complete today is described in +[Get Started](../get-started.md) and the [guides](../guides/index.md). +::: + **An AgentVisor is the hypervisor for Agent execution.** It organizes compute, filesystems, networks, models, tools, credentials, and diff --git a/docs/src/en/pvisor/concepts/index.md b/docs/src/en/pvisor/concepts/index.md index 70c11dde..47b58174 100644 --- a/docs/src/en/pvisor/concepts/index.md +++ b/docs/src/en/pvisor/concepts/index.md @@ -12,14 +12,14 @@ providers make different guarantees. Follow these articles in order: -1. [What is an AgentVisor?](agentvisor.md) explains the product category and - the boundary between an Agent and its runtime. -2. [Run, Attempt, and Effect](run-model.md) defines the stable objects that - survive process and provider changes. -3. [Capabilities and evidence](capabilities-and-evidence.md) explains how a +1. [Run, Attempt, and Effect](run-model.md) defines one execution, its attempts, + and the changes it produces. +2. [Capabilities and evidence](capabilities-and-evidence.md) explains how a request becomes an installed mechanism and a claim in the Run Bundle. +3. [What is an AgentVisor?](agentvisor.md) explains the product category and + the boundary between an Agent and its runtime. -The category article is implementation-neutral. The Run and capability pages +AgentVisor is a category definition, not a pVisor feature list. The Run and capability pages define pVisor's stable user model. Platform mechanisms and current gaps belong to [pVisor Design](../design/index.md). diff --git a/docs/src/en/pvisor/guides/capture.md b/docs/src/en/pvisor/guides/capture.md index 27b6e8ee..76203660 100644 --- a/docs/src/en/pvisor/guides/capture.md +++ b/docs/src/en/pvisor/guides/capture.md @@ -5,18 +5,7 @@ there is no standalone Gateway command or daemon. The [capability and evidence model](../concepts/capabilities-and-evidence.md) explains what capture proves and what it does not enforce. -## Local walkthrough - -```bash -cargo build --release -p persisting-pvisor --bin pvisor -cd examples/pvisor/04-gateway-llm-control -./run.sh -``` - -The example starts a loopback OpenAI-compatible model, executes its Agent with -`pvisor run`, and prints Gateway counters and the captured conversation. - -## Run a real Agent +Install `pvisor` using the [installation guide](../../installation.md), then run a real Agent: ```bash export DEEPSEEK_API_KEY=sk-... diff --git a/docs/src/en/pvisor/index.md b/docs/src/en/pvisor/index.md index 0dba4480..411f3c5b 100644 --- a/docs/src/en/pvisor/index.md +++ b/docs/src/en/pvisor/index.md @@ -7,13 +7,14 @@ environment.** It gives each Run its own workspace boundary, records the controls that were actually installed, and lets you review filesystem changes before they reach the project. -Within Persisting's model-state-to-Agent-history story, pVisor owns the -execution boundary and the reviewable record of one Run. +In Persisting, pVisor runs one Agent and reviews its changes. You can use it +without pChronicle. :::tip What you will complete -By the end of the first walkthrough you will have a Run-owned stage, a Run -Bundle that explains the controls actually installed, and a deliberate choice -to apply or drop the resulting filesystem effects. +By the end of the first walkthrough: the Agent has stopped; its changes remain +in a Run-owned staging directory; you deliberately write them into the project +or discard them. Your first `pvisor review` shows the record of controls that +were actually installed. ::: pVisor does not replace the Agent's reasoning loop. You can keep using Agent diff --git a/docs/src/en/rfcs/0013-pchronicle-warehouse-catalog.md b/docs/src/en/rfcs/0013-pchronicle-warehouse-catalog.md index 637f44d7..0247b79e 100644 --- a/docs/src/en/rfcs/0013-pchronicle-warehouse-catalog.md +++ b/docs/src/en/rfcs/0013-pchronicle-warehouse-catalog.md @@ -19,10 +19,17 @@ CLI 标志、配置文件和 HTTP 路径为兼容性仍使用 `catalog` 一词 规范实现挂在现有 `pchronicle serve --catalog-config` 上,不引入独立 `catalog serve` 进程,也不把 listener 从 loopback 打开。 -- **CLI**:`@team` 解析为 `catalog://127.0.0.1:PORT` Directory locator;`@team/prod` 换票后客户端打开票里的 path(透传后端密钥)。 -- **Web**:用户钥存在浏览器 `localStorage`;数据面查询在 **新进程 worker** 中执行。父进程负责鉴权和换票,不拿全量后端密钥跑 DataFusion。 +- **Serve 挂载**:`pchronicle serve --catalog-config FILE` MUST 把 `catalog.toml` 中的 **全部** + `[datasets.*]` 挂进 Warehouse(与位置参数挂载等价)。本机 Web / 无用户钥的数据面请求在 + **父进程内**打开这些 mount,不再 front-only。 +- **CLI 配置**:`pchronicle serve catalog dataset add|remove|list` 改写 libraries; + `issue|grant|revoke` 改写用户与授权。 +- **Directory 换票**:`@team` 解析为 `catalog://…`;`@team/prod` 换票后客户端打开票里的 path。 + `/api/v1/catalog/datasets` 仍按用户钥过滤可见 library。 ```text +pchronicle serve catalog dataset add --catalog-config catalog.toml prod --uri s3://bucket/prod \ + --access-key BACKEND_AK --secret-key BACKEND_SK pchronicle serve catalog issue --catalog-config catalog.toml alice pchronicle serve catalog grant --catalog-config catalog.toml alice prod evals pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 @@ -154,6 +161,9 @@ datasets = ["evals"] 签发和改授权是 **写 `catalog.toml` 的 CLI**,不是运行中 Warehouse 的 HTTP API。出现 `catalog` 子命令时 MUST NOT 启动 listener。正在运行的 serve MUST 重启后才读到新用户或新授权。 ```text +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [--endpoint URL] [--region REGION] [--access-key KEY] [--secret-key KEY] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... +pchronicle serve catalog dataset list --catalog-config FILE pchronicle serve catalog issue --catalog-config FILE NAME pchronicle serve catalog grant --catalog-config FILE NAME DATASET... pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... @@ -161,6 +171,14 @@ pchronicle serve --catalog-config FILE --listen 127.0.0.1:8081 ``` `catalog` 是 `serve` 的保留子命令。要挂载名为 `catalog` 的路径,使用 `./catalog` 或 `NAME=./catalog`。 +`--catalog-config` MUST NOT 与位置参数 Dataset 同时使用。 + +### `dataset add` / `remove` / `list` + +- MUST NOT 启动 Warehouse。只改 `FILE` 后退出。 +- `add` 写入 `[datasets.NAME]`。已存在的名字 MUST 拒绝。`s3://` MUST 设置后端钥,且 MUST 与文件中已有 s3 library 的 endpoint/region/ak/sk 完全一致;非 `s3://` MUST NOT 设置后端钥。 +- `remove` 删除列出的 library。若仍有 grant 引用该 library,MUST 失败且 MUST NOT 改文件。 +- `list` 打印 `name` / `uri`(及可选 endpoint/region),MUST NOT 打印后端密钥。 ### `issue` diff --git a/docs/src/en/rfcs/0014-compact-jsonl.md b/docs/src/en/rfcs/0014-compact-jsonl.md index 5119d281..9925d063 100644 --- a/docs/src/en/rfcs/0014-compact-jsonl.md +++ b/docs/src/en/rfcs/0014-compact-jsonl.md @@ -51,16 +51,20 @@ Storyline,也不推断轨迹语义。 ### 文件集合 -显式文件输入 MUST 是扩展名大小写不敏感的 `.jsonl` 普通文件。目录输入 MUST 递归选择 -`.jsonl` 普通文件,并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 +显式文件输入 MUST 是扩展名大小写不敏感的 `.json`、`.jsonl` 或 `.ndjson` 普通文件。目录输入 +MUST 递归选择这些普通文件,并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 字节序处理文件;单个文件内部 MUST 按原始行顺序处理。 -每个被选择的文件 MUST 至少包含一条 record。每一物理行 MUST: +`.jsonl` 和 `.ndjson` 文件中的每一物理行 MUST: 1. 不是空行或纯空白行; 2. 是 UTF-8 JSON; 3. 顶层值是 JSON object; -4. 包含可映射为非空标量的 `id` 和 `timestamp`。 +4. 包含可映射为非空标量的 `timestamp`;缺失或无效的 `id` 和 `timestamp` 使用稳定的 + `source_filename#line_number` 值生成。 + +`.json` 文件 MUST 是一个 object 或 array,并作为一条 record 导入;数组保持为 record 的 JSON +值,不会按元素拆分。导出时保留该 JSON document。 任一条件不满足时,整个 import/sync MUST 失败,不得发布部分 dataset。 @@ -78,7 +82,9 @@ index = "[" 1*DIGIT "]" 示例:`$`、`$.id`、`$.user.id`、`$.messages[0].role`。 不支持 wildcard、slice、filter、递归下降、quoted member、负数 index 或一段中的多个 index。 -映射缺失时,附加列写 Arrow null;`id` 或 `timestamp` 映射缺失时必须拒绝该 record。 +映射缺失时,附加列写 Arrow null;`id` 或 `timestamp` 映射缺失或无效时,使用该 record 的 +规范化 source filename 和 1-based line number 生成值,格式为 `source_filename#line_number`。 +生成的值只写入 Lance 基础列,不修改 `_raw_` 或 `data`。 ### 列映射 @@ -95,9 +101,9 @@ timestamp=$.timestamp `NAME` MUST 符合上述 `identifier`。映射名 MUST 唯一。`filename`、`data`、`_raw_` 和 `_offload_` 是保留名,MUST NOT 用作用户映射名。 -`id` 和 `timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入;number 使用 -JSON 规范表示写为 UTF-8。null、boolean、array、object 与空 string 均无效。dataset 内的 -`id` MUST 唯一。 +`timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入;number 使用 JSON +规范表示写为 UTF-8。有效的 `id` 来源值也按同样规则写入;其他 `id` 值触发上述生成规则。 +dataset 内的 `id` MUST 唯一。 ## Lance 物理 schema diff --git a/docs/src/en/rfcs/0015-chronicle-manifest.md b/docs/src/en/rfcs/0015-chronicle-manifest.md new file mode 100644 index 00000000..600e4b08 --- /dev/null +++ b/docs/src/en/rfcs/0015-chronicle-manifest.md @@ -0,0 +1,277 @@ +# RFC-0015: `chronicle.manifest` Dataset Sidecar + +| Field | Value | +|---|---| +| **Status** | Proposed | +| **Format name** | `chronicle.manifest` (TOML) | +| **Date** | 2026-09-07 | +| **Component** | `persisting-pchronicle`, `pchronicle` CLI, Warehouse explorer | +| **Implements** | `crates/persisting-pchronicle/src/store/chronicle_manifest.rs` | +| **Related** | [RFC-0014 Compact JSONL](0014-compact-jsonl.md) · [RFC-0013 path Directory](0013-pchronicle-warehouse-catalog.md) · [RFC-0001 Storyline](0001-storyline-format.md) | + +--- + +## Summary + +`chronicle.manifest` is a **pChronicle-owned TOML sidecar** placed at a Dataset +node root. It enables cheap discovery and stores common aggregate statistics so +Warehouse catalog / explorer paths do not need to open Lance datasets or scan +every record on every refresh. + +This RFC uses RFC 2119 **MUST**, **MUST NOT**, **SHOULD**, and **MAY**. + +## Motivation + +Large local datasets (for example a compact-jsonl Lance directory with many +rows and `_offload/` objects) currently force discovery to `Dataset::open` and +force explorer acceleration to materialize per-record run summaries. With a +warehouse that mounts several such roots, five-second catalog refresh and tree +polling keep the serve process on high CPU even when the operator only browses +folder counts. + +Lance's internal `_versions/*.manifest` is an MVCC control plane owned by Lance. +pChronicle MUST NOT encode application discovery or UI statistics there. + +pChronicle already uses application control files for other layouts (`CURRENT` +for Storyline, `_manifest.json` for Events). `chronicle.manifest` extends that +pattern to a uniform, nestable Dataset node descriptor. + +## Goals and non-goals + +Goals: + +- Define a stable on-disk file name, TOML schema, and nesting rules. +- Let discovery classify Dataset nodes by reading a small TOML file. +- Persist aggregate stats used by explorer tree / dataset summaries. +- Support nested Dataset trees by **automatically scanning** child directories for + `chronicle.manifest`. +- Keep missing or stale manifests compatible with existing heuristic discovery. + +Non-goals (v1): + +- Extending Lance protobuf manifests. +- Replacing SQL or detailed run/record listing with the sidecar. +- Hand-maintained explicit `children` lists in parent manifests. +- Rewriting Storyline `CURRENT` or Events `_manifest.json` into this format + (those remain authoritative for their layouts; optional future alignment). + +## Terminology + +- **Dataset node**: a directory that is either a physical source root (leaf) or + a directory that aggregates nested Dataset nodes (branch). +- **Leaf**: a node whose `format` identifies a physical store (v1: + `compact-jsonl/v1`). +- **Branch**: a node that exists to nest children; it has no physical `format`. +- **Fingerprint**: a string that binds `[stats]` to one physical revision so + readers can detect staleness. + +## File location and name + +- The file name MUST be exactly `chronicle.manifest`. +- The file MUST live at the Dataset node root (sibling to Lance `data/`, + Storyline `CURRENT`, etc., when those exist). +- Encoding MUST be UTF-8 TOML. + +## Nesting and discovery + +### Automatic child scan + +Parents MUST NOT require an explicit children list. Discovery MUST: + +1. If the current directory contains `chronicle.manifest`, parse it. +2. If `kind = "leaf"`, treat the directory as one source candidate for + `format` and MUST NOT recurse into its interior for additional sources. +3. If `kind = "branch"`, scan **immediate** child directories only; for each + child that contains `chronicle.manifest`, treat that child as a nested + Dataset node and continue according to that child's kind. +4. If the current directory has no `chronicle.manifest`, keep the existing + heuristic discovery, but when a subdirectory contains + `chronicle.manifest`, prefer that node and MUST NOT open Lance solely to + classify it. + +Symlinks MUST be ignored. Existing `max_entries` / `max_files` limits still +apply to traversal. + +### Branch aggregation and trajectory counts + +Branch nodes MAY omit `[stats]` and MUST NOT be treated as trajectory sources. +Only leaf nodes contribute trajectories. + +When presenting Catalog / explorer folder totals (`run_count` / +`record_count`): + +- Readers MUST compute a prefix total as the **sum of all descendant leaf** + `[stats].record_count` (and `failed_count`) values discovered under that + prefix. +- Intermediate branch directories contribute **0** of their own; they only + define nesting. +- A leaf MUST NOT recurse for additional nested sources, so a physical leaf + cannot double-count child leaves. + +#### No ancestor write-back (write amplification) + +Publishing or updating a leaf MUST update **only** that leaf's +`chronicle.manifest`. Writers MUST NOT rewrite ancestor branch manifests to +cache rolled-up totals. Branch files SHOULD remain descriptive only, for +example: + +```toml +schema_version = 1 +kind = "branch" +``` + +Rolled-up folder counts are a **read-side** concern. + +#### Process-level refresh cache + +Warehouse / Catalog MAY keep an in-process cache of discovered leaf stats and +prefix aggregates (for example on the existing catalog snapshot / +acceleration path) so periodic UI refresh does not re-open Lance or re-list +large trees. Cache entries SHOULD invalidate when a leaf `fingerprint` or +manifest mtime changes, or when a new `chronicle.manifest` appears under a +scanned prefix. Process cache MUST NOT replace on-disk leaf manifests as the +source of truth that travels with the dataset. + +## TOML schema (v1) + +### Required top-level fields + +| Field | Type | Rules | +|---|---|---| +| `schema_version` | integer | MUST be `1` for this RFC | +| `kind` | string | MUST be `"leaf"` or `"branch"` | + +### Leaf-only fields + +| Field | Type | Rules | +|---|---|---| +| `format` | string | MUST be present for `kind = "leaf"`; v1 writers MUST use `compact-jsonl/v1` | + +Unknown `format` values MUST be preserved by generic readers; format-specific +openers MAY reject unsupported values. + +### `[identity]` + +| Field | Type | Rules | +|---|---|---| +| `fingerprint` | string | MUST be present when `[stats]` is present; binds stats to a physical revision | + +For compact-jsonl v1, fingerprint SHOULD be `lance:version:` where `` is +the published Lance dataset version after write. + +### `[stats]` + +| Field | Type | Rules | +|---|---|---| +| `record_count` | integer ≥ 0 | MUST be present for leaf compact-jsonl writers | +| `failed_count` | integer ≥ 0 | MUST be present; use `0` when unknown/none | +| `min_timestamp` | string | MAY be omitted | +| `max_timestamp` | string | MAY be omitted | +| `total_tokens` | integer ≥ 0 | MAY be omitted | + +Additional stats keys MAY be added in later schema versions; v1 readers MUST +ignore unknown keys under `[stats]`. + +### Example: leaf + +```toml +schema_version = 1 +kind = "leaf" +format = "compact-jsonl/v1" + +[identity] +fingerprint = "lance:version:1" + +[stats] +record_count = 12345 +failed_count = 0 +min_timestamp = "2026-01-01T00:00:00Z" +max_timestamp = "2026-09-07T01:00:00Z" +``` + +### Example: branch + +```toml +schema_version = 1 +kind = "branch" +``` + +## Write path + +- `chronicle.manifest` is a **store-layer contract**. The only compact-jsonl + publication exits are `CompactJsonlStore::publish_manifest` and + `CompactJsonlStore::import_path`. CLI `import` and `sync` MUST go through that + store API and MUST NOT invent a parallel sidecar writer. +- Compact JSONL `import` / successful republish / `sync` snapshot MUST write + `chronicle.manifest` at the output dataset root. +- Writes MUST be atomic on local filesystems (write temp + rename into place). +- After a successful physical write, `fingerprint` MUST match the published + revision and `[stats].record_count` MUST equal the published row count. +- If manifesto publication fails during `import_path`, the import MUST fail so + a half-published contract is not exposed. For read-side `ensure_manifest` + upgrades, failure MAY be logged while still opening the physical dataset. + +## Read path and staleness + +- When `fingerprint` matches the opened physical revision, readers MAY trust + `[stats]` for explorer aggregates without scanning rows. +- When the file is missing, unreadable, or fingerprint mismatches, store-layer + `ensure_manifest` SHOULD rewrite the sidecar in place; if that fails, readers + MUST fall back to existing discovery / summary paths. +- Manifest stats MUST NOT be the sole authority for query correctness; SQL and + record listing still read the physical store. + +## Warehouse explorer implications + +- Catalog refresh and `/api/explorer/tree` SHOULD use nested manifests for + discovery and folder `run_count` / `record_count` aggregates when available. +- Folder `run_count` at any prefix MUST equal the sum of descendant leaf + trajectory weights (manifest `record_count` when present), not the count of + child dataset nodes. +- Detailed run/record pages MAY still open the physical leaf; this RFC does + not require a full sidecar index of every record identity. + +## Required unit tests (v1) + +Implementations MUST cover at least: + +1. **Nested discovery**: `warehouse/(branch)` → `team/(branch)` → + `codex_jsonl/(leaf, N)` yields exactly one compact source at + `team/codex_jsonl` with `record_count = N`, without opening Lance when the + leaf manifesto is present. +2. **Sibling leaves**: two leaves under one branch with counts `A` and `B` + yield two sources; tree root `run_count = A + B`; each child folder shows + its own leaf total. +3. **Prefix roll-up**: under dataset scope, prefix `team` aggregates all + leaves under `team/…`; prefix equal to a leaf path shows that leaf only. +4. **Leaf non-recursion**: a leaf directory that also contains a nested + `chronicle.manifest` MUST NOT emit an additional source for the nested + child. +5. **Write isolation (contract)**: updating one leaf's manifesto MUST NOT + require changing parent branch files for counts to remain correct after + rediscovery (read-side sum). + +## Compatibility + +- Datasets without `chronicle.manifest` remain valid. The first store open or + heuristic discovery that confirms compact-jsonl SHOULD backfill the sidecar. +- Lance schema metadata `pchronicle.format = compact-jsonl/v1` remains the + physical format marker; the sidecar does not replace it. +- Object-store URIs are out of scope for v1 writers; remote reads MAY be added + later with the same schema. Nested branch scanning on object stores MAY use + prefix listing plus exact-key reads of `chronicle.manifest`; v1 does not + require S3 name-glob search. + +## Alternatives considered + +1. **Extend Lance `_versions/*.manifest`** — rejected: binary MVCC format, + not owned by pChronicle, unsuitable for nesting and UI stats. +2. **Warehouse-only memory cache as the sole stats store** — rejected: does + not travel with the dataset and resets on process restart. Process cache + remains valid as a **refresh optimization** on top of on-disk leaf + manifests. +3. **Explicit parent `children` lists** — deferred: automatic scanning matches + directory trees and avoids stale child lists. +4. **Write rolled-up `[stats]` onto every ancestor branch** — rejected for + v1: causes write amplification and stale parents when a single leaf is + updated. diff --git a/docs/src/en/rfcs/index.md b/docs/src/en/rfcs/index.md index a4f09b76..d68ca2b5 100644 --- a/docs/src/en/rfcs/index.md +++ b/docs/src/en/rfcs/index.md @@ -20,3 +20,4 @@ each product's Reference and Guides. | [0012](0012-pchronicle-find-query-syntax.md) | pChronicle `find` query expression | Proposed | | [0013](0013-pchronicle-warehouse-catalog.md) | pChronicle path Directory | Proposed | | [0014](0014-compact-jsonl.md) | Compact JSONL Lance storage format (`compact-jsonl/v1`) | Accepted | +| [0015](0015-chronicle-manifest.md) | `chronicle.manifest` Dataset sidecar (TOML) | Proposed | diff --git a/docs/src/en/system-design/index.md b/docs/src/en/system-design/index.md index 0b49627d..461cc5e7 100644 --- a/docs/src/en/system-design/index.md +++ b/docs/src/en/system-design/index.md @@ -1,7 +1,7 @@ # System Design -Persisting is persistent infrastructure for the Agent era, spanning model -state—parameters and KV caches—and Agent history. This section focuses on the +Persisting provides durable infrastructure for Agent execution and trajectory +history. This section focuses on the current public product path: - [pVisor](../pvisor/index.md) virtualizes and governs one Agent Run; diff --git a/docs/src/index.md b/docs/src/index.md index 7b61dbac..cd3333ec 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,7 +1,7 @@ --- template: home.html title: Persisting — Persistent Infrastructure for the Agent Era -description: Persisting model state and Agent history—from model parameters and KV caches to Agent trajectories. +description: Run Agents under a reviewable execution boundary and preserve queryable history. hide: toc --- diff --git a/docs/src/zh/index.md b/docs/src/zh/index.md index b8f86d99..024f1d52 100644 --- a/docs/src/zh/index.md +++ b/docs/src/zh/index.md @@ -7,20 +7,18 @@ sidebar_label: 从这里开始 Persisting 提供两条独立的产品路径,请按当前任务选择入口: -- [使用 pVisor 安全运行 Agent](pvisor/get-started.md):暂存 workspace 修改,检查 Run Bundle,只应用你批准的内容。 -- [使用 pChronicle 探索持久历史](pchronicle/get-started.md):打开 Dataset,执行只读查询,并追踪 Source lineage。 -- [理解产品边界](overview.md):了解执行与历史如何连接,同时保持边界清晰。 +- [使用 pVisor 安全运行 Agent](pvisor/get-started.md):在暂存工作区里跑 Agent,检查改动,只把你批准的内容写入项目。 +- [使用 pChronicle 探索持久历史](pchronicle/get-started.md):打开一份轨迹数据,跑只读查询,弄清查的是哪份数据、哪个来源。 +- [选择工作流](overview.md):弄清该走哪条路径,以及执行与历史如何可选地连接。 如果你正在评估 Persisting,先阅读[选择工作流](overview.md),再进入对应的产品快速开始。 ## 完成第一次 walkthrough 后你会得到什么 -- **pVisor** walkthrough 会以停止的 Run、可读取的 Run Bundle,以及明确的 apply 或 drop - 决策结束。只有你选择 apply,项目才会收到 staged Effect。 -- **pChronicle** walkthrough 会以一次只读 Dataset 查询结束,并明确区分正在读取的 Dataset、 - Source 和 Snapshot。 +- **pVisor**:Agent 已停止;改动仍在暂存目录;你明确选择写入项目或丢弃。只有你选择写入,项目才会变化。 +- **pChronicle**:你对一份轨迹数据跑过只读查询,并知道查的是哪份数据、哪个来源。 -开始时不需要同时使用两个产品。只有在需要把执行证据与持久轨迹历史关联起来时,才配置 capture 交接。 +开始时不需要同时使用两个产品。只有在需要把一次执行与持久轨迹关联起来时,再配置 capture 交接。 ## 开始前准备 diff --git a/docs/src/zh/overview.md b/docs/src/zh/overview.md index 27e08394..204e8408 100644 --- a/docs/src/zh/overview.md +++ b/docs/src/zh/overview.md @@ -4,16 +4,15 @@ Persisting 有两个独立入口。先选择与你当前任务相符的入口, ## 我想运行 Agent,并审查它产生的修改 -从 **pVisor** 开始。它为单个 Agent 提供 Run 独占 workspace,记录执行边界,并把文件修改 -留在 stage 中,直到你决定哪些内容进入真实项目。 +从 **pVisor** 开始。它让单个 Agent 在独立 Run 中工作,记录执行边界,并把文件修改 +留在暂存区,直到你决定写入真实项目还是丢弃。 1. [安装命令行工具](installation.md)。 2. [运行第一个 Agent](pvisor/get-started.md)。 3. [审查并选择性应用修改](pvisor/guides/review-apply.md)。 4. [选择 host、OCI 或 VM 执行环境](pvisor/guides/execution.md)。 -完成后,你应该得到一个已结束的 Run、一份可读的 Run Bundle,以及一个已经 apply 或 drop 的 -stage。 +完成后,Agent 已停止,改动留在暂存区,你可以审查后写入或丢弃。 ```bash pvisor run --stage ./runs/task-001 -- codex @@ -31,7 +30,7 @@ pvisor apply last --path src 3. [导入或导出支持的格式](pchronicle/guides/exchange.md)。 4. [在本地提供 Dataset 服务](pchronicle/guides/serve.md)。 -完成后,你应该能执行只读查询,看到规范化视图,并明确所查看数据的 Source lineage。 +完成后,你应该能执行只读查询,看到规范化视图,并清楚查的是哪份数据、哪个来源。 ```bash pchronicle onboard query @@ -45,8 +44,10 @@ pchronicle query ./trajectory-data \ lifecycle record 发布到 pChronicle。这个交接是显式且有限的:它不会搬运私有 Run Bundle,也 不会补造原始 Source 没有提供的 Evidence。 +pVisor capture 与 `pchronicle serve --gateway` 是两条入口:前者随 Agent Run 启停并共享执行边界;后者独立接收或转发已有 Agent/SDK 的流量,不启动 pVisor Run。 + 1. [捕获 Agent 轨迹](pvisor/guides/capture.md)。 -2. [理解 event 与 sidecar 契约](rfcs/0007-events-contract-pchronicle-sidecar.md)。 +2. [理解 event 与 sidecar 契约](rfcs/0007-events-contract-pchronicle-sidecar.md)(需要改协议或排障时再读)。 3. [阅读从执行到历史的架构](system-design/architecture.md)。 ```text diff --git a/docs/src/zh/pchronicle/index.md b/docs/src/zh/pchronicle/index.md index 764dded9..71302f51 100644 --- a/docs/src/zh/pchronicle/index.md +++ b/docs/src/zh/pchronicle/index.md @@ -5,12 +5,12 @@ **pChronicle 是 Agent 轨迹存储引擎。** 用于浏览、查询、交换和服务运行 Dataset;既可以读取 Persisting 产生的运行记录,也可以直接读取受支持的外部格式;不要求先运行 pVisor。 -在 Persisting“从模型状态到 Agent 历史”的主线中,pChronicle 是 Agent 历史的持久存储与查询引擎。 -它可以作为本地工具使用,也可以平台化部署在多条 path 前面。 +在 Persisting 里,pChronicle 负责保存与查询轨迹历史;不要求先跑 pVisor。 +它可以作为本地工具使用,也可以在多条 path 前面以服务方式部署。 :::tip 你将完成什么 -第一次快速开始会创建临时数据,将它打开为 Dataset,运行一次只读摘要,再回答一个 SQL 问题。 -你不需要先准备生产存储,就能熟悉查询模型。 +第一次快速开始会创建临时数据,打开它,跑一次只读摘要,再回答一个 SQL 问题。 +你不需要先准备生产存储。 ::: ## 你只需要面对 Dataset diff --git a/docs/src/zh/pchronicle/reference/cli.md b/docs/src/zh/pchronicle/reference/cli.md index d317417c..daacab2c 100644 --- a/docs/src/zh/pchronicle/reference/cli.md +++ b/docs/src/zh/pchronicle/reference/cli.md @@ -330,8 +330,9 @@ Storyline Dataset。默认 `create` 模式要求目标不存在。`append` 要 Compact JSONL 是记录存储,不会转换或推断轨迹语义。指定 `--input-format compact-jsonl` 或 `--output-format compact-jsonl` 均会选择该格式。输入必须是本地 -`.jsonl` 文件或目录树;每个非空行必须是 JSON object,并包含唯一的标量 `id` 和标量 -`timestamp`,默认路径分别为 `$.id` 和 `$.timestamp`。`--column id=PATH` 与 +`.json`、`.jsonl` 或 `.ndjson` 文件或目录树;JSON object 和 array 都会被接受,每个 JSON 文档 +会成为一条 record,数组会完整保留。缺失或无效的 `id` 和 `timestamp` 会获得稳定的 +`source_filename#line_number` 值,默认路径分别为 `$.id` 和 `$.timestamp`。`--column id=PATH` 与 `--column timestamp=PATH` 用于覆盖默认路径,其他 `--column NAME=PATH` 会增加 nullable JSONB 投影列。Compact import 支持本地 `create` 和经确认的 `replace`,不支持 stdin、对象存储目标或 `append`。 @@ -350,7 +351,7 @@ pchronicle sync --from DIRECTORY --to DIRECTORY --convert DIRECTORY 变更并指数退避重试。`--once` 只执行一次初始批次后退出。当前目标必须是本地目录,两个目标 必须位于源目录之外。 -指定 `--input-format compact-jsonl` 时,源目录必须是本地 `.jsonl` 目录树,列映射规则与 Compact +指定 `--input-format compact-jsonl` 时,源目录必须是本地 `.json`、`.jsonl` 或 `.ndjson` 目录树,列映射规则与 Compact import 相同。每个成功批次都会重新扫描整个目录,并原子替换 `--convert` 指向的 Compact Lance 快照,因此新增、修改和删除都会反映在下一快照中,但不提供行级增量更新。此模式仍要求传入 `--to` 作为兼容参数,但不会写入该路径。 diff --git a/docs/src/zh/project/index.md b/docs/src/zh/project/index.md index ea75ca3e..33dadecf 100644 --- a/docs/src/zh/project/index.md +++ b/docs/src/zh/project/index.md @@ -1,7 +1,7 @@ # Project -Persisting 的公共定位横跨模型状态与 Agent 历史。这一节记录交付状态、稳定决策、贡献者 -工作流,以及不在当前 pVisor 与 pChronicle 主路径中的独立系统。 +Persisting 的公开产品主路径是 pVisor 与 pChronicle。这一节记录交付状态、稳定决策、贡献者 +工作流,以及不在当前主路径中的独立系统。 ## 架构 diff --git a/docs/src/zh/pvisor/concepts/agentvisor.md b/docs/src/zh/pvisor/concepts/agentvisor.md index e9465175..4ee1f4b6 100644 --- a/docs/src/zh/pvisor/concepts/agentvisor.md +++ b/docs/src/zh/pvisor/concepts/agentvisor.md @@ -1,5 +1,10 @@ # 什么是 AgentVisor? +:::note 阅读说明 +本文定义 **AgentVisor 品类**,不是 pVisor 的功能清单或交付承诺。当前可完成的能力以 +[快速开始](../get-started.md) 与 [指南](../guides/index.md) 为准。 +::: + **AgentVisor 是虚拟化 Agent 执行的 Hypervisor。** 它把个人电脑、工作站或集群中的计算、文件系统、网络、模型、工具、凭据和持久状态 diff --git a/docs/src/zh/pvisor/concepts/index.md b/docs/src/zh/pvisor/concepts/index.md index 1310fe35..10992380 100644 --- a/docs/src/zh/pvisor/concepts/index.md +++ b/docs/src/zh/pvisor/concepts/index.md @@ -10,12 +10,12 @@ Provider 的保证不同,就从这里开始。 请按以下顺序阅读: -1. [什么是 AgentVisor?](agentvisor.md) 解释产品类别,以及 Agent 与运行时之间的边界。 -2. [Run、Attempt 与 Effect](run-model.md) 定义可以跨进程和 Provider 保留的稳定对象。 -3. [Capability 与 Evidence](capabilities-and-evidence.md) 解释请求如何变成实际机制,以及如何 +1. [Run、Attempt 与 Effect](run-model.md) 定义一次执行、它的尝试和产生的修改。 +2. [Capability 与 Evidence](capabilities-and-evidence.md) 解释请求如何变成实际机制,以及如何 写入 Run Bundle。 +3. [什么是 AgentVisor?](agentvisor.md) 解释 pVisor 所属的产品类别,以及 Agent 与运行时之间的边界。 -品类文章不绑定具体实现;Run 和 capability 文章定义 pVisor 稳定的用户模型;平台机制与 +AgentVisor 是品类定义,不是 pVisor 的功能清单;Run 和 capability 文章定义 pVisor 稳定的用户模型;平台机制与 当前缺口属于 [pVisor Design](../design/index.md)。 读完本节后,你应该能查看 Run Bundle,并区分请求的 capability 与实际生效的 capability。 diff --git a/docs/src/zh/pvisor/guides/capture.md b/docs/src/zh/pvisor/guides/capture.md index 08e40cb4..d31bff92 100644 --- a/docs/src/zh/pvisor/guides/capture.md +++ b/docs/src/zh/pvisor/guides/capture.md @@ -4,13 +4,7 @@ Gateway capture 是 pVisor 的 Run 驱动,由 Run 启停;系统不再提供 守护进程。[Capability 与 Evidence 模型](../concepts/capabilities-and-evidence.md)解释 Capture 能证明什么,以及它不负责 enforce 什么。 -```bash -cargo build --release -p persisting-pvisor --bin pvisor -cd examples/pvisor/04-gateway-llm-control -./run.sh -``` - -真实 Agent 可直接通过 `pvisor run` 配置: +先按[安装指南](../../installation.md)安装 `pvisor`;本页直接使用已安装的命令。真实 Agent 可直接通过 `pvisor run` 配置: ```bash export DEEPSEEK_API_KEY=sk-... diff --git a/docs/src/zh/pvisor/index.md b/docs/src/zh/pvisor/index.md index aad8b177..2cd85f6b 100644 --- a/docs/src/zh/pvisor/index.md +++ b/docs/src/zh/pvisor/index.md @@ -5,11 +5,11 @@ **pVisor 在受控执行环境中运行现有的 Agent 命令。** 它为每个 Run 提供独立的工作区边界, 记录实际生效的控制机制,并让你在文件变更进入项目之前先进行审查。 -在 Persisting“从模型状态到 Agent 历史”的主线中,pVisor 负责一个 Run 的执行边界与可审查记录。 +在 Persisting 里,pVisor 负责跑一次 Agent 并审查其改动;可与 pChronicle 分开使用。 :::tip 你将完成什么 -完成第一次快速开始后,你会得到一个由 Run 拥有的 stage、一份解释实际控制机制的 Run Bundle, -以及一次明确的选择:应用或丢弃产生的 filesystem Effect。 +完成第一次快速开始后:Agent 已停止;改动留在 Run 独占的暂存目录;你明确选择写入项目或丢弃。 +第一次 `pvisor review` 时,你会看到解释实际控制机制的记录。 ::: pVisor 不替代 Agent 自己的推理循环。你可以继续使用已有的 Agent CLI、脚本和 framework。 diff --git a/docs/src/zh/rfcs/0013-pchronicle-warehouse-catalog.md b/docs/src/zh/rfcs/0013-pchronicle-warehouse-catalog.md index 6333e0c8..f70ba3a3 100644 --- a/docs/src/zh/rfcs/0013-pchronicle-warehouse-catalog.md +++ b/docs/src/zh/rfcs/0013-pchronicle-warehouse-catalog.md @@ -19,10 +19,17 @@ CLI 标志、配置文件和 HTTP 路径为兼容性仍使用 `catalog` 一词 规范实现挂在现有 `pchronicle serve --catalog-config` 上,不引入独立 `catalog serve` 进程,也不把 listener 从 loopback 打开。 -- **CLI**:`@team` 解析为 `catalog://127.0.0.1:PORT` Directory locator;`@team/prod` 换票后客户端打开票里的 path(透传后端密钥)。 -- **Web**:用户钥存在浏览器 `localStorage`;数据面查询在 **新进程 worker** 中执行。父进程负责鉴权和换票,不拿全量后端密钥跑 DataFusion。 +- **Serve 挂载**:`pchronicle serve --catalog-config FILE` MUST 把 `catalog.toml` 中的 **全部** + `[datasets.*]` 挂进 Warehouse(与位置参数挂载等价)。本机 Web / 无用户钥的数据面请求在 + **父进程内**打开这些 mount,不再 front-only。 +- **CLI 配置**:`pchronicle serve catalog dataset add|remove|list` 改写 libraries; + `issue|grant|revoke` 改写用户与授权。 +- **Directory 换票**:`@team` 解析为 `catalog://…`;`@team/prod` 换票后客户端打开票里的 path。 + `/api/v1/catalog/datasets` 仍按用户钥过滤可见 library。 ```text +pchronicle serve catalog dataset add --catalog-config catalog.toml prod --uri s3://bucket/prod \ + --access-key BACKEND_AK --secret-key BACKEND_SK pchronicle serve catalog issue --catalog-config catalog.toml alice pchronicle serve catalog grant --catalog-config catalog.toml alice prod evals pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 @@ -154,21 +161,16 @@ permissions = ["read", "query", "analyze"] Catalog 管理命令只修改配置文件,不启动 HTTP listener。文件不存在时,命令创建父目录和空配置文件。 ```text -pchronicle serve catalog user create --catalog-config FILE NAME -pchronicle serve catalog user list --catalog-config FILE -pchronicle serve catalog user remove --catalog-config FILE NAME - -pchronicle serve catalog dataset create --catalog-config FILE NAME URI [OPTIONS] +pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] +pchronicle serve catalog dataset remove --catalog-config FILE NAME... pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog dataset show --catalog-config FILE NAME -pchronicle serve catalog dataset remove --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE USER DATASET --permission PERMISSION... -pchronicle serve catalog revoke --catalog-config FILE USER DATASET --permission PERMISSION... -pchronicle serve catalog grants --catalog-config FILE +pchronicle serve catalog issue --catalog-config FILE NAME +pchronicle serve catalog grant --catalog-config FILE NAME DATASET... +pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... ``` -`user create` 生成用户 AK/SK;secret 只在本次 stdout 输出。`dataset create` 只登记 Dataset,不创建或删除后端数据。`grant` 和 `revoke` 修改独立的 `[[grants]]` 授权记录。所有写操作 MUST 原子替换文件,失败时保留原文件。 +`dataset add` 只登记 Dataset,不创建或删除后端数据。`issue` 生成用户 AK/SK(secret 只在本次 stdout 输出)。`grant` / `revoke` 修改 `[[grants]]`。所有写操作 MUST 原子替换文件,失败时保留原文件。`serve --catalog-config` MUST 挂载文件中全部 datasets。 ## HTTP diff --git a/docs/src/zh/rfcs/0014-compact-jsonl.md b/docs/src/zh/rfcs/0014-compact-jsonl.md index 5119d281..7bf073bb 100644 --- a/docs/src/zh/rfcs/0014-compact-jsonl.md +++ b/docs/src/zh/rfcs/0014-compact-jsonl.md @@ -51,16 +51,20 @@ Storyline,也不推断轨迹语义。 ### 文件集合 -显式文件输入 MUST 是扩展名大小写不敏感的 `.jsonl` 普通文件。目录输入 MUST 递归选择 -`.jsonl` 普通文件,并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 +显式文件输入 MUST 是扩展名大小写不敏感的 `.json`、`.jsonl` 或 `.ndjson` 普通文件。目录输入 +MUST 递归选择这些普通文件,并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 字节序处理文件;单个文件内部 MUST 按原始行顺序处理。 -每个被选择的文件 MUST 至少包含一条 record。每一物理行 MUST: +`.jsonl` 和 `.ndjson` 文件中的每一物理行 MUST: 1. 不是空行或纯空白行; 2. 是 UTF-8 JSON; 3. 顶层值是 JSON object; -4. 包含可映射为非空标量的 `id` 和 `timestamp`。 +4. 包含可映射为非空标量的 `timestamp`;缺失或无效的 `id` 和 `timestamp` 使用稳定的 + `source_filename#line_number` 值生成。 + +`.json` 文件必须是一个 object 或 array,并作为一条 record 导入;数组会作为 record 的 JSON +值完整保留,不会按元素拆分。导出时保留该 JSON document。 任一条件不满足时,整个 import/sync MUST 失败,不得发布部分 dataset。 @@ -78,7 +82,9 @@ index = "[" 1*DIGIT "]" 示例:`$`、`$.id`、`$.user.id`、`$.messages[0].role`。 不支持 wildcard、slice、filter、递归下降、quoted member、负数 index 或一段中的多个 index。 -映射缺失时,附加列写 Arrow null;`id` 或 `timestamp` 映射缺失时必须拒绝该 record。 +映射缺失时,附加列写 Arrow null;`id` 或 `timestamp` 映射缺失或无效时,使用该 record 的 +规范化 source filename 和从 1 开始的行号生成值,格式为 `source_filename#line_number`。 +生成的值只写入 Lance 基础列,不修改 `_raw_` 或 `data`。 ### 列映射 @@ -95,9 +101,9 @@ timestamp=$.timestamp `NAME` MUST 符合上述 `identifier`。映射名 MUST 唯一。`filename`、`data`、`_raw_` 和 `_offload_` 是保留名,MUST NOT 用作用户映射名。 -`id` 和 `timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入;number 使用 -JSON 规范表示写为 UTF-8。null、boolean、array、object 与空 string 均无效。dataset 内的 -`id` MUST 唯一。 +`timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入;number 使用 JSON +规范表示写为 UTF-8。有效的 `id` 来源值也按同样规则写入;其他 `id` 值触发上述生成规则。 +dataset 内的 `id` MUST 唯一。 ## Lance 物理 schema diff --git a/docs/src/zh/rfcs/0015-chronicle-manifest.md b/docs/src/zh/rfcs/0015-chronicle-manifest.md new file mode 100644 index 00000000..a66250de --- /dev/null +++ b/docs/src/zh/rfcs/0015-chronicle-manifest.md @@ -0,0 +1,230 @@ +# RFC-0015: `chronicle.manifest` Dataset Sidecar + +| Field | Value | +|---|---| +| **Status** | Proposed | +| **Format name** | `chronicle.manifest`(TOML) | +| **Date** | 2026-09-07 | +| **Component** | `persisting-pchronicle`、`pchronicle` CLI、Warehouse explorer | +| **Implements** | `crates/persisting-pchronicle/src/store/chronicle_manifest.rs` | +| **Related** | [RFC-0014 Compact JSONL](0014-compact-jsonl.md) · [RFC-0013 path Directory](0013-pchronicle-warehouse-catalog.md) · [RFC-0001 Storyline](0001-storyline-format.md) | + +--- + +## 摘要 + +`chronicle.manifest` 是放在 Dataset 节点根目录的 **pChronicle 自有 TOML sidecar**。 +它用于廉价发现,并保存常用聚合统计,使 Warehouse catalog / explorer 不必在每次刷新时 +打开 Lance 或扫描全部记录。 + +本文使用 RFC 2119 的 **MUST**、**MUST NOT**、**SHOULD** 与 **MAY**。 + +## 动机 + +大型本地数据集(例如含大量行与 `_offload/` 的 compact-jsonl Lance 目录)目前会迫使 +discovery 执行 `Dataset::open`,并迫使 explorer acceleration 物化逐条 run 摘要。当 +Warehouse 挂载多个此类根目录时,约五秒一次的 catalog 刷新与 tree 轮询会让 serve 进程 +持续高 CPU,即便用户只在浏览文件夹计数。 + +Lance 内部的 `_versions/*.manifest` 是 Lance 的 MVCC 控制面。pChronicle MUST NOT 在其中 +编码应用层发现或 UI 统计。 + +pChronicle 已对其它布局使用应用控制文件(Storyline 的 `CURRENT`、Events 的 +`_manifest.json`)。`chronicle.manifest` 把该模式扩展为可嵌套的 Dataset 节点描述符。 + +## 目标与非目标 + +目标: + +- 定义稳定的文件名、TOML schema 与嵌套规则; +- 让 discovery 通过读小 TOML 文件即可分类 Dataset 节点; +- 持久化 explorer tree / dataset 摘要所需的聚合统计; +- 通过**自动扫描**子目录中的 `chronicle.manifest` 支持嵌套 Dataset 树; +- 在 sidecar 缺失或过期时,仍兼容现有启发式发现。 + +非目标(v1): + +- 扩展 Lance protobuf manifest; +- 用 sidecar 替代 SQL 或详细 run/record 列表; +- 在父 manifest 中手写显式 `children` 列表; +- 把 Storyline `CURRENT` 或 Events `_manifest.json` 改写成此格式(它们仍是各自布局的权威;可选后续对齐)。 + +## 术语 + +- **Dataset 节点**:作为物理 source 根(leaf)或聚合嵌套 Dataset 节点(branch)的目录。 +- **Leaf**:`format` 标识物理存储的节点(v1:`compact-jsonl/v1`)。 +- **Branch**:用于嵌套子节点、没有物理 `format` 的节点。 +- **Fingerprint**:把 `[stats]` 绑定到某一物理修订的字符串,供读者检测过期。 + +## 文件位置与名称 + +- 文件名 MUST 恰好为 `chronicle.manifest`。 +- 文件 MUST 位于 Dataset 节点根(与 Lance `data/`、Storyline `CURRENT` 等并列)。 +- 编码 MUST 为 UTF-8 TOML。 + +## 嵌套与发现 + +### 自动扫描子节点 + +父节点 MUST NOT 要求显式 children 列表。Discovery MUST: + +1. 若当前目录存在 `chronicle.manifest`,则解析它; +2. 若 `kind = "leaf"`,将该目录视为对应 `format` 的一个 source 候选,且 MUST NOT 再递归其内部寻找其它 source; +3. 若 `kind = "branch"`,只扫描**一层**子目录;对每个含有 `chronicle.manifest` 的子目录,按该子节点的 kind 继续处理; +4. 若当前目录没有 `chronicle.manifest`,保留现有启发式发现,但当子目录含有 `chronicle.manifest` 时,优先采用该节点,且 MUST NOT 仅为分类而打开 Lance。 + +MUST 忽略符号链接。现有 `max_entries` / `max_files` 遍历上限仍然适用。 + +### Branch 聚合与轨迹计数 + +Branch 节点 MAY 省略 `[stats]`,且 MUST NOT 被当成轨迹 source。 +只有 leaf 贡献轨迹数。 + +Catalog / explorer 展示文件夹合计(`run_count` / `record_count`)时: + +- 读者 MUST 将某一 path 前缀的总数算为该前缀下**所有子孙 leaf** 的 + `[stats].record_count`(以及 `failed_count`)之和; +- 中间 branch 自身贡献 **0**,只负责嵌套; +- leaf MUST NOT 再向下递归寻找嵌套 source,避免物理 leaf 与子 leaf 双计。 + +#### 禁止祖先回写(写放大) + +发布或更新某个 leaf 时,MUST **只**更新该 leaf 的 `chronicle.manifest`。 +写入方 MUST NOT 为缓存累计总数而改写祖先 branch manifest。Branch 文件 SHOULD +仅作描述,例如: + +```toml +schema_version = 1 +kind = "branch" +``` + +文件夹累计数属于**读侧**职责。 + +#### 进程内刷新缓存 + +Warehouse / Catalog MAY 在进程内缓存已发现的 leaf stats 与前缀聚合(例如挂在 +现有 catalog snapshot / acceleration 路径上),使周期性 UI 刷新不必重开 Lance +或重扫大目录树。当 leaf 的 `fingerprint` / manifest mtime 变化,或扫描前缀下 +出现新的 `chronicle.manifest` 时,缓存条目 SHOULD 失效。进程缓存 MUST NOT +取代随数据一起分发的磁盘 leaf manifest 作为真相源。 + +## TOML schema(v1) + +### 顶层必填 + +| 字段 | 类型 | 规则 | +|---|---|---| +| `schema_version` | integer | 本 RFC MUST 为 `1` | +| `kind` | string | MUST 为 `"leaf"` 或 `"branch"` | + +### 仅 Leaf + +| 字段 | 类型 | 规则 | +|---|---|---| +| `format` | string | `kind = "leaf"` 时 MUST 存在;v1 写入方 MUST 使用 `compact-jsonl/v1` | + +未知 `format` 值 MUST 被通用读者保留;特定格式 opener MAY 拒绝不支持的值。 + +### `[identity]` + +| 字段 | 类型 | 规则 | +|---|---|---| +| `fingerprint` | string | 存在 `[stats]` 时 MUST 存在;把 stats 绑定到物理修订 | + +compact-jsonl v1 的 fingerprint SHOULD 为 `lance:version:`,其中 `` 是写入后发布的 +Lance dataset version。 + +### `[stats]` + +| 字段 | 类型 | 规则 | +|---|---|---| +| `record_count` | integer ≥ 0 | leaf compact-jsonl 写入方 MUST 提供 | +| `failed_count` | integer ≥ 0 | MUST 提供;未知/无失败时用 `0` | +| `min_timestamp` | string | MAY 省略 | +| `max_timestamp` | string | MAY 省略 | +| `total_tokens` | integer ≥ 0 | MAY 省略 | + +后续 schema 版本 MAY 增加更多 stats 键;v1 读者 MUST 忽略 `[stats]` 下的未知键。 + +### Leaf 示例 + +```toml +schema_version = 1 +kind = "leaf" +format = "compact-jsonl/v1" + +[identity] +fingerprint = "lance:version:1" + +[stats] +record_count = 12345 +failed_count = 0 +min_timestamp = "2026-01-01T00:00:00Z" +max_timestamp = "2026-09-07T01:00:00Z" +``` + +### Branch 示例 + +```toml +schema_version = 1 +kind = "branch" +``` + +## 写路径 + +- `chronicle.manifest` 是 **store 层标准机制**:compact-jsonl 的唯一发布出口是 + `CompactJsonlStore::publish_manifest` / `import_path`。CLI `import` 与 `sync` + MUST 只通过该 store API,不得在上层另写并行 sidecar。 +- Compact JSONL `import` / 成功 republish / `sync` snapshot MUST 在输出 dataset 根写入 + `chronicle.manifest`。 +- 本机文件系统上的写入 MUST 原子(写临时文件再 rename)。 +- 物理写入成功后,`fingerprint` MUST 匹配已发布修订,且 `[stats].record_count` MUST 等于已发布行数。 +- 若 dataset 写成功但 manifest 写失败,`import_path` MUST 失败(不发布半成品契约);对 + 仅 `ensure_manifest` 的只读升级路径,失败 MAY 记日志并继续打开物理数据。 + +## 读路径与过期 + +- 当 `fingerprint` 与已打开物理修订一致时,读者 MAY 信任 `[stats]` 做 explorer 聚合,而无需扫行。 +- 文件缺失、不可读或 fingerprint 不匹配时,store 层 `ensure_manifest` SHOULD 就地补写; + 若补写失败,读者 MUST 回退现有 discovery / summary 路径。 +- Manifest stats MUST NOT 成为查询正确性的唯一权威;SQL 与 record 列表仍读物理存储。 + +## 对 Warehouse explorer 的影响 + +- Catalog 刷新与 `/api/explorer/tree` 在可用时应优先用嵌套 manifest 做发现与文件夹 + `run_count` / `record_count` 聚合。 +- 任意前缀上的文件夹 `run_count` MUST 等于其下子孙 leaf 轨迹权重之和(有 manifesto 时用 + `record_count`),而不是子 dataset 节点个数。 +- 详细 run/record 页仍可打开物理 leaf;本 RFC 不要求 sidecar 索引每条 record 身份。 + +## 必需单元测试(v1) + +实现 MUST 至少覆盖: + +1. **嵌套发现**:`warehouse/(branch)` → `team/(branch)` → `codex_jsonl/(leaf, N)` + 恰好得到一条 compact source,路径为 `team/codex_jsonl`,`record_count = N`,且在 + leaf manifesto 存在时不打开 Lance。 +2. **兄弟 leaf**:同一 branch 下两个 leaf,计数分别为 `A`、`B`,得到两条 source; + tree 根 `run_count = A + B`;各子文件夹显示各自 leaf 总量。 +3. **前缀上卷**:在 dataset 作用域内,前缀 `team` 聚合 `team/…` 下全部 leaf;前缀等于 + 某 leaf 路径时只显示该 leaf。 +4. **Leaf 不递归**:leaf 目录内即使还有嵌套 `chronicle.manifest`,也 MUST NOT 再为该 + 子节点额外产出 source。 +5. **写隔离(契约)**:只更新某一个 leaf 的 manifesto 后,重新 discovery 仍能通过读侧 + 求和得到正确祖先总数,且无需改动父 branch 文件。 + +## 兼容性 + +- 没有 `chronicle.manifest` 的旧 dataset 仍然有效;首次经 store 打开或启发式 discovery + 确认 compact-jsonl 时,SHOULD 自动补写 sidecar。 +- Lance schema metadata `pchronicle.format = compact-jsonl/v1` 仍是物理格式标记;sidecar 不替代它。 +- 对象存储 URI 不在 v1 写入范围内;远端读取可在以后用同一 schema 扩展。对象存储上的嵌套 + branch 扫描 MAY 使用前缀列举 + 精确读取 `chronicle.manifest`;v1 不要求 S3 按文件名 glob 搜索。 + +## 曾考虑的替代方案 + +1. **扩展 Lance `_versions/*.manifest`** — 否决:二进制 MVCC、非 pChronicle 所有、不适合嵌套与 UI 统计。 +2. **仅 Warehouse 内存缓存作为唯一统计存储** — 否决:不随数据走,进程重启即失效。进程缓存 + 仍可作为磁盘 leaf manifesto 之上的**刷新优化**。 +3. **父节点显式 `children` 列表** — 延后:自动扫描更贴合目录树,也避免子列表过期。 +4. **把累计 `[stats]` 回写到每个祖先 branch** — v1 否决:单 leaf 更新会写放大,且易产生脏父节点。 diff --git a/docs/src/zh/rfcs/index.md b/docs/src/zh/rfcs/index.md index bf7ee436..b35b8b96 100644 --- a/docs/src/zh/rfcs/index.md +++ b/docs/src/zh/rfcs/index.md @@ -21,3 +21,4 @@ RFC 正文保持英文,因为它们是历史决策快照。翻译会产生两 | [0012](0012-pchronicle-find-query-syntax.md) | pChronicle `find` 查询表达式 | Proposed | | [0013](0013-pchronicle-warehouse-catalog.md) | pChronicle path Directory | Proposed | | [0014](0014-compact-jsonl.md) | Compact JSONL Lance 存储格式(`compact-jsonl/v1`) | Accepted | +| [0015](0015-chronicle-manifest.md) | `chronicle.manifest` Dataset sidecar(TOML) | Proposed | diff --git a/docs/src/zh/system-design/index.md b/docs/src/zh/system-design/index.md index db393d7c..81f6f3b5 100644 --- a/docs/src/zh/system-design/index.md +++ b/docs/src/zh/system-design/index.md @@ -1,6 +1,6 @@ # System Design -Persisting 是横跨模型状态——参数与 KV Cache——以及 Agent 历史的持久化基础设施。本节聚焦 +Persisting 提供 Agent 执行与轨迹历史的持久化基础设施。本节聚焦 当前公开产品路径: - [pVisor](../pvisor/index.md) 虚拟化并治理单个 Agent Run; diff --git a/pchronicle-web/assets/catalog.css b/pchronicle-web/assets/catalog.css index e6d8a623..f19955b5 100644 --- a/pchronicle-web/assets/catalog.css +++ b/pchronicle-web/assets/catalog.css @@ -11,20 +11,19 @@ .pc-catalog-stats strong{color:#101828;font:600 18px ui-monospace,SFMono-Regular,Menlo,monospace} .pc-catalog-errors{margin:0 0 10px;color:#b42318;font-size:11px} .pc-catalog-mosaic{min-height:0;flex:1;display:flex;flex-direction:column} -.pc-catalog-tree{position:relative;min-height:0;flex:1;border:1px solid #dfe3e8;border-radius:12px;background:#eef2f6;overflow:hidden} -.pc-catalog-tree.compact{flex:0 0 auto;display:flex;flex-wrap:wrap;gap:10px;padding:12px;overflow:visible} -.pc-catalog-tree.compact .pc-catalog-tile{position:static;width:260px;height:120px} -.pc-catalog-tile{position:absolute;display:flex;flex-direction:column;align-items:flex-start;justify-content:space-between;padding:10px;border:1px solid #ffffffaa;border-radius:8px;color:#102033;text-align:left;cursor:pointer;overflow:hidden} -.pc-catalog-tile strong{max-width:100%;overflow:hidden;font-size:13px;text-overflow:ellipsis;white-space:nowrap} -.pc-catalog-tile small{color:#1f2937cc;font:11px ui-monospace,SFMono-Regular,Menlo,monospace} -.pc-catalog-tile.tone-0{background:#93c5fd} -.pc-catalog-tile.tone-1{background:#6ea8ff} -.pc-catalog-tile.tone-2{background:#67e8f9} -.pc-catalog-tile.tone-3{background:#86efac} -.pc-catalog-tile.tone-4{background:#fde68a} -.pc-catalog-tile.tone-5{background:#e2e8f0} -.pc-catalog-tile.kind-other{background:#cbd5e1} -.pc-catalog-tile:hover,.pc-catalog-tile:focus-visible{outline:0;box-shadow:inset 0 0 0 2px #1d4ed8} +.pc-catalog-folders{display:grid;grid-template-columns:repeat(auto-fill,minmax(230px,1fr));gap:12px;min-height:220px;padding:14px;border:1px solid #dfe3e8;border-radius:12px;background:#f8fafc;overflow:auto} +.pc-catalog-folder{display:flex;min-height:132px;flex-direction:column;align-items:flex-start;justify-content:space-between;padding:14px;border:1px solid #d0d5dd;border-radius:10px;background:#fff;color:#102033;text-align:left;cursor:pointer;box-shadow:0 1px 2px #1018280d} +.pc-catalog-folder-title{display:flex;align-items:center;gap:9px;width:100%;min-width:0} +.pc-catalog-folder-title strong{overflow:hidden;font-size:14px;text-overflow:ellipsis;white-space:nowrap} +.pc-catalog-folder-icon{display:grid;width:28px;height:24px;flex:none;place-items:center;border-radius:6px;background:#dbeafe;color:#2563eb;font-size:16px} +.pc-catalog-folder-type{padding:3px 7px;border-radius:999px;background:#eef2ff;color:#4338ca;font:700 9px ui-monospace,SFMono-Regular,Menlo,monospace} +.pc-catalog-folder-meta{display:flex;justify-content:space-between;gap:12px;width:100%;color:#667085;font:11px ui-monospace,SFMono-Regular,Menlo,monospace} +.pc-catalog-folder.type-compact-jsonl{border-top:3px solid #06b6d4} +.pc-catalog-folder.type-storyline{border-top:3px solid #2563eb} +.pc-catalog-folder.type-mixed{border-top:3px solid #8b5cf6} +.pc-catalog-folder.type-other{border-top:3px solid #f59e0b} +.pc-catalog-folder.kind-other{background:#f1f5f9} +.pc-catalog-folder:hover,.pc-catalog-folder:focus-visible{outline:0;box-shadow:0 0 0 2px #1d4ed8,0 4px 12px #10182814} .pc-catalog-empty{min-height:220px;flex:1;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:6px;border:1px dashed #d0d5dd;border-radius:12px;color:#98a2b3;text-align:center} .pc-catalog-empty strong{color:#475467;font-size:13px} .pc-catalog-empty span{font-size:11px} diff --git a/pchronicle-web/assets/components.css b/pchronicle-web/assets/components.css index e3935ff4..82c5f6aa 100644 --- a/pchronicle-web/assets/components.css +++ b/pchronicle-web/assets/components.css @@ -474,47 +474,187 @@ } .pc2-json-scalar { + display: block; + max-width: 100%; + line-height: 1.45; white-space: pre-wrap; word-break: break-word; color: #344054; } +.pc2-json-root { + min-width: 0; + font: 12px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; +} + .pc2-json-tree { + min-width: 0; display: flex; flex-direction: column; - gap: 2px; + gap: 0; } .pc2-json-node { - border-left: 1px solid #e4e7ec; - padding-left: 8px; + min-width: 0; +} + +.pc2-json-leaf { + display: flex; + align-items: baseline; + gap: 7px; + min-width: 0; + min-height: 22px; + padding: 1px 3px; } +.pc2-json-root > summary, .pc2-json-node > summary { display: flex; align-items: center; - gap: 8px; + gap: 7px; + min-height: 24px; + padding: 1px 3px; cursor: pointer; list-style: none; } +.pc2-json-root > summary::before, +.pc2-json-node > summary::before { + box-sizing: border-box; + display: inline-flex; + width: 14px; + height: 14px; + flex: 0 0 14px; + align-items: center; + justify-content: center; + border: 1px solid #94a3b8; + border-radius: 2px; + background: linear-gradient(#fff, #e2e8f0); + color: #475569; + content: "+"; + font: 700 11px/12px sans-serif; +} + +.pc2-json-root[open] > summary::before, +.pc2-json-node[open] > summary::before { + content: "−"; +} + +.pc2-json-root > summary:hover, +.pc2-json-node > summary:hover { + background: #f1f5f9; +} + +.pc2-json-root > summary::-webkit-details-marker, .pc2-json-node > summary::-webkit-details-marker { display: none; } +.pc2-json-root > .pc2-json-tree, +.pc2-json-node > .pc2-json-tree { + margin-left: 7px; + padding-left: 20px; + border-left: 1px dotted #cbd5e1; +} + +.pc2-json-kind { + color: #4338ca; + font-size: 15px; + font-weight: 800; +} + +.pc2-json-leaf-icon { + box-sizing: border-box; + width: 9px; + height: 9px; + flex: 0 0 9px; + border: 1px solid #cbd5e1; + box-shadow: inset 0 0 0 1px #ffffff80; +} + +.pc2-json-leaf-icon.string { + background: #334ea2; +} + +.pc2-json-leaf-icon.number { + background: #2f8f46; +} + +.pc2-json-leaf-icon.boolean { + background: #d4b106; +} + +.pc2-json-leaf-icon.null { + background: #c9362b; +} + .pc2-json-key { color: #101828; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 11px; } -.pc2-json-size { +.pc2-json-punctuation { color: #667085; - font-size: 9px; +} + +.pc2-json-value { + min-width: 0; + white-space: pre-wrap; + word-break: break-word; +} + +.pc2-json-value.string { + color: #067647; +} + +.pc2-json-value.number { + color: #175cd3; +} + +.pc2-json-value.boolean { + color: #b54708; +} + +.pc2-json-value.null { + color: #98a2b3; +} + +.pc2-json-long-value { + min-width: 0; + flex: 1; +} + +.pc2-json-long-value > summary { + overflow: hidden; + cursor: pointer; + list-style: none; + text-overflow: ellipsis; + white-space: nowrap; +} + +.pc2-json-long-value > summary::-webkit-details-marker { + display: none; +} + +.pc2-json-long-value > summary::before { + margin-right: 4px; + color: #98a2b3; + content: "…"; +} + +.pc2-json-expanded-value { + max-height: 280px; + margin-top: 4px; + overflow: auto; + padding: 6px 8px; + border-left: 2px solid #d0d5dd; + background: #f8fafc; + white-space: pre-wrap; + word-break: break-word; } .pc2-json-node > .pc2-json-table, -.pc2-json-node > .pc2-json-tree, .pc2-json-node > .pc2-json-scroll, .pc2-json-node > .pc2-json-scalar { margin: 6px 0 8px; diff --git a/pchronicle-web/src/catalog.rs b/pchronicle-web/src/catalog.rs index 93188844..d3167b35 100644 --- a/pchronicle-web/src/catalog.rs +++ b/pchronicle-web/src/catalog.rs @@ -2,66 +2,6 @@ use dioxus::prelude::*; use crate::model::{CatalogTree, CatalogTreeChild}; -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct TileBox { - pub x: f64, - pub y: f64, - pub w: f64, - pub h: f64, -} - -pub fn layout_treemap(sizes: &[f64], width: f64, height: f64) -> Vec { - if sizes.is_empty() || width <= 0.0 || height <= 0.0 { - return Vec::new(); - } - let total: f64 = sizes.iter().copied().sum(); - if total <= 0.0 { - return sizes - .iter() - .map(|_| TileBox { - x: 0.0, - y: 0.0, - w: 0.0, - h: 0.0, - }) - .collect(); - } - split(sizes, 0.0, 0.0, width, height) -} - -fn split(areas: &[f64], x: f64, y: f64, w: f64, h: f64) -> Vec { - match areas { - [] => Vec::new(), - [_] => vec![TileBox { x, y, w, h }], - _ => { - let total: f64 = areas.iter().sum(); - let mut acc = 0.0; - let mut cut = 1; - for (index, area) in areas.iter().enumerate() { - acc += area; - cut = index + 1; - if acc >= total / 2.0 { - break; - } - } - cut = cut.clamp(1, areas.len() - 1); - let left_sum: f64 = areas[..cut].iter().sum(); - let frac = left_sum / total; - if w >= h { - let left = w * frac; - let mut tiles = split(&areas[..cut], x, y, left, h); - tiles.extend(split(&areas[cut..], x + left, y, w - left, h)); - tiles - } else { - let top = h * frac; - let mut tiles = split(&areas[..cut], x, y, w, top); - tiles.extend(split(&areas[cut..], x, y + top, w, h - top)); - tiles - } - } - } -} - #[component] pub fn CatalogExplorer( tree: Option, @@ -111,7 +51,7 @@ pub fn CatalogExplorer( span { "This path contains one source file. Open it in Runs to inspect its runs." } } } else { - CatalogMosaic { + CatalogFolders { tree: tree.clone().unwrap(), on_open, on_runs, @@ -208,46 +148,23 @@ fn CatalogStats(tree: Option) -> Element { } #[component] -fn CatalogMosaic( +fn CatalogFolders( tree: CatalogTree, on_open: EventHandler<(String, String)>, on_runs: EventHandler<(String, String)>, on_other: EventHandler, ) -> Element { - let sizes = tree - .children - .iter() - .map(|child| child.run_count.max(1) as f64) - .collect::>(); - let boxes = layout_treemap(&sizes, 100.0, 100.0); let dataset = tree.dataset.clone().unwrap_or_default(); - // A treemap with one or two children stretches a single tile across the - // whole viewport. Render those as fixed-size cards instead. - let compact = tree.children.len() <= 2; - let tree_class = if compact { - "pc-catalog-tree compact" - } else { - "pc-catalog-tree" - }; rsx! { - div { class: "{tree_class}", - for (index, child) in tree.children.iter().cloned().enumerate() { - { - let tile = boxes.get(index).copied().unwrap_or(TileBox { x: 0.0, y: 0.0, w: 0.0, h: 0.0 }); - let tone = index % 6; - rsx! { - CatalogTile { - key: "{child.kind}:{child.path}", - child, - dataset: dataset.clone(), - tile, - tone, - compact, - on_open, - on_runs, - on_other, - } - } + div { class: "pc-catalog-folders", + for child in tree.children.iter().cloned() { + CatalogFolder { + key: "{child.kind}:{child.path}", + child, + dataset: dataset.clone(), + on_open, + on_runs, + on_other, } } } @@ -255,32 +172,22 @@ fn CatalogMosaic( } #[component] -fn CatalogTile( +fn CatalogFolder( child: CatalogTreeChild, dataset: String, - tile: TileBox, - tone: usize, - compact: bool, on_open: EventHandler<(String, String)>, on_runs: EventHandler<(String, String)>, on_other: EventHandler, ) -> Element { - let style = if compact { - String::new() - } else { - format!( - "left:{:.3}%;top:{:.3}%;width:{:.3}%;height:{:.3}%;", - tile.x, tile.y, tile.w, tile.h - ) - }; let kind = child.kind.clone(); let path = child.path.clone(); let name = child.name.clone(); + let data_type = child.data_type.clone(); + let tokens = format_tokens(child.total_tokens); rsx! { button { - class: "pc-catalog-tile tone-{tone} kind-{kind}", - style, - title: "{name} · {child.run_count} runs", + class: "pc-catalog-folder type-{data_type} kind-{kind}", + title: "{name} · {child.run_count} trajectories · {tokens} tokens", onclick: move |event| { match kind.as_str() { "other" => on_other.call(event), @@ -289,8 +196,15 @@ fn CatalogTile( _ => on_open.call((dataset.clone(), path.clone())), } }, - strong { "{child.name}" } - small { "{child.run_count}" } + div { class: "pc-catalog-folder-title", + span { class: "pc-catalog-folder-icon", if child.kind == "file" { "▤" } else { "▰" } } + strong { "{child.name}" } + } + span { class: "pc-catalog-folder-type", "{data_type}" } + div { class: "pc-catalog-folder-meta", + span { "{child.run_count} trajectories" } + span { "{tokens} tokens" } + } } } } @@ -366,22 +280,3 @@ fn format_tokens(tokens: Option) -> String { tokens.to_string() } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn treemap_covers_the_canvas_and_keeps_area_proportional() { - let boxes = layout_treemap(&[2.0, 1.0, 1.0], 100.0, 100.0); - assert_eq!(boxes.len(), 3); - let area: f64 = boxes.iter().map(|tile| tile.w * tile.h).sum(); - assert!((area - 10_000.0).abs() < 0.01); - assert!((boxes[0].w * boxes[0].h - 5_000.0).abs() < 0.01); - } - - #[test] - fn empty_sizes_yield_no_tiles() { - assert!(layout_treemap(&[], 100.0, 100.0).is_empty()); - } -} diff --git a/pchronicle-web/src/json_value.rs b/pchronicle-web/src/json_value.rs index fc3504ca..37deb22c 100644 --- a/pchronicle-web/src/json_value.rs +++ b/pchronicle-web/src/json_value.rs @@ -1,6 +1,8 @@ use dioxus::prelude::*; use serde_json::Value; +const JSON_VALUE_PREVIEW_LIMIT: usize = 240; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum JsonShape { Scalar, @@ -74,17 +76,6 @@ pub fn record_columns(rows: &[Value]) -> Vec { columns } -pub fn json_summary(value: &Value) -> String { - match peel_json(value) { - Value::Object(object) => format!("{{{} keys}}", object.len()), - Value::Array(items) => format!("[{} items]", items.len()), - Value::String(_) => "string".into(), - Value::Number(_) => "number".into(), - Value::Bool(_) => "boolean".into(), - Value::Null => "null".into(), - } -} - fn scalar_text(value: &Value) -> String { match value { Value::Null => "null".into(), @@ -95,6 +86,42 @@ fn scalar_text(value: &Value) -> String { } } +fn json_literal(value: &Value) -> String { + serde_json::to_string(value).unwrap_or_else(|_| scalar_text(value)) +} + +fn json_preview(value: &str) -> String { + let mut chars = value.chars(); + let preview = chars + .by_ref() + .take(JSON_VALUE_PREVIEW_LIMIT) + .collect::(); + if chars.next().is_some() { + format!("{preview}…") + } else { + preview + } +} + +fn json_type(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +fn json_kind_icon(value: &Value) -> &'static str { + match value { + Value::Array(_) => "[]", + Value::Object(_) => "{}", + _ => "", + } +} + #[component] pub fn JsonValue(value: Value, #[props(default = false)] default_open: bool) -> Element { let peeled = peel_json(&value); @@ -109,6 +136,37 @@ pub fn JsonValue(value: Value, #[props(default = false)] default_open: bool) -> } } +#[component] +pub fn JsonViewer(value: Value) -> Element { + let peeled = peel_json(&value); + let kind = json_type(&peeled); + rsx! { + details { class: "pc2-json-root {kind}", open: true, + summary { + span { class: "pc2-json-kind", "{json_kind_icon(&peeled)}" } + strong { "JSON" } + } + JsonTree { value: peeled, default_open: false } + } + } +} + +#[component] +fn JsonScalar(value: Value) -> Element { + let literal = json_literal(&value); + let kind = json_type(&value); + if literal.chars().count() > JSON_VALUE_PREVIEW_LIMIT { + rsx! { + details { class: "pc2-json-long-value", + summary { class: "pc2-json-value {kind}", "{json_preview(&literal)}" } + div { class: "pc2-json-expanded-value {kind}", "{literal}" } + } + } + } else { + rsx! { span { class: "pc2-json-value {kind}", "{literal}" } } + } +} + #[component] fn JsonKvTable(value: Value, default_open: bool) -> Element { let map = match value { @@ -168,11 +226,7 @@ fn JsonRecordTable(value: Value, default_open: bool) -> Element { #[component] fn JsonTree(value: Value, default_open: bool) -> Element { match value { - Value::Array(items) if items.is_empty() => rsx! { - details { class: "pc2-json-node", open: default_open, - summary { span { class: "pc2-json-size", "[0 items]" } } - } - }, + Value::Array(items) if items.is_empty() => rsx! { div { class: "pc2-json-tree" } }, Value::Object(map) => rsx! { div { class: "pc2-json-tree", for (key, child) in map { @@ -183,7 +237,7 @@ fn JsonTree(value: Value, default_open: bool) -> Element { Value::Array(items) => rsx! { div { class: "pc2-json-tree", for (index, child) in items.into_iter().enumerate() { - JsonTreeNode { key: "{index}", label: format!("[{index}]"), value: child, default_open } + JsonTreeNode { key: "{index}", label: String::new(), value: child, default_open: false } } } }, @@ -196,11 +250,24 @@ fn JsonTree(value: Value, default_open: bool) -> Element { #[component] fn JsonTreeNode(label: String, value: Value, default_open: bool) -> Element { - let summary = json_summary(&value); + let peeled = peel_json(&value); + let kind = json_type(&peeled); + if is_scalar(&peeled) { + return rsx! { + div { class: "pc2-json-leaf", + span { class: "pc2-json-leaf-icon {kind}" } + if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } + JsonScalar { value: peeled } + } + }; + } rsx! { - details { class: "pc2-json-node", open: default_open, - summary { span { class: "pc2-json-key", "{label}" } span { class: "pc2-json-size", "{summary}" } } - JsonValue { value, default_open } + details { class: "pc2-json-node {kind}", open: default_open, + summary { + span { class: "pc2-json-kind", "{json_kind_icon(&peeled)}" } + if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } + } + JsonTree { value: peeled, default_open: false } } } } @@ -269,14 +336,11 @@ mod tests { } #[test] - fn json_summary_peels_and_names_types() { - assert_eq!(json_summary(&json!({"a": 1, "b": 2})), "{2 keys}"); - assert_eq!(json_summary(&json!([1, 2, 3])), "[3 items]"); - assert_eq!(json_summary(&json!([])), "[0 items]"); - assert_eq!(json_summary(&json!("{}")), "{0 keys}"); - assert_eq!(json_summary(&json!("hello")), "string"); - assert_eq!(json_summary(&json!(true)), "boolean"); - assert_eq!(json_summary(&json!(1)), "number"); - assert_eq!(json_summary(&json!(null)), "null"); + fn long_json_values_get_single_line_previews() { + let value = "x".repeat(JSON_VALUE_PREVIEW_LIMIT + 1); + let preview = json_preview(&value); + assert_eq!(preview.chars().count(), JSON_VALUE_PREVIEW_LIMIT + 1); + assert!(preview.ends_with('…')); + assert_eq!(json_preview("short"), "short"); } } diff --git a/pchronicle-web/src/model.rs b/pchronicle-web/src/model.rs index f1c6ae03..a6a9a4dd 100644 --- a/pchronicle-web/src/model.rs +++ b/pchronicle-web/src/model.rs @@ -137,12 +137,16 @@ pub struct CatalogTreeChild { pub name: String, pub kind: String, #[serde(default)] + pub data_type: String, + #[serde(default)] pub path: String, #[serde(default)] pub run_count: usize, #[serde(default)] pub failed_count: usize, #[serde(default)] + pub total_tokens: Option, + #[serde(default)] pub entries: Vec, } diff --git a/pchronicle-web/src/workspace.rs b/pchronicle-web/src/workspace.rs index 0c2223ea..eba95370 100644 --- a/pchronicle-web/src/workspace.rs +++ b/pchronicle-web/src/workspace.rs @@ -18,7 +18,7 @@ use crate::copilot_sessions::{ page_after_history_switch, persist_indexed_thread, relative_time, restore_for_run, save_index, session_storage_key, title_from_thread, }; -use crate::json_value::JsonValue; +use crate::json_value::JsonViewer; use crate::llm; use crate::llm_settings::LlmSettings; #[cfg(test)] @@ -32,6 +32,7 @@ use crate::notice::{ErrorNotice, WorkspaceNotice, workspace_notice}; use crate::terminology::{ANALYSIS, ASSISTANT, DATASETS, RUNS, STEPS, STORAGE, TIMELINE}; const SEARCH_DEBOUNCE_MS: u32 = 1_000; +const CATALOG_REFRESH_MS: u32 = 5_000; fn evidence_notice(turn_id: i64, detail: &str) -> WorkspaceNotice { WorkspaceNotice { @@ -201,13 +202,31 @@ pub fn App() -> Element { if page() != "catalog" { return; } + let dataset = catalog_dataset(); + let prefix = catalog_prefix(); load_catalog_tree( - catalog_dataset(), - catalog_prefix(), + dataset.clone(), + prefix.clone(), catalog_tree, catalog_loading, error, ); + spawn(async move { + loop { + TimeoutFuture::new(CATALOG_REFRESH_MS).await; + if page() != "catalog" || catalog_dataset() != dataset || catalog_prefix() != prefix + { + break; + } + load_catalog_tree( + dataset.clone(), + prefix.clone(), + catalog_tree, + catalog_loading, + error, + ); + } + }); }); use_effect(move || { @@ -1488,7 +1507,7 @@ fn RunDetailWorkspace( if loading && compact_record.is_none() { div { class: "pc2-inline-loading", span { class: "spinner" } "Loading record…" } } else if let Some(detail) = compact_record { - JsonValue { value: detail.record, default_open: true } + JsonViewer { value: detail.record } } else { div { class: "pc2-empty", strong { "Record unavailable" } } }