From 8d4967870ed6cd38dbe57c4d94e4a2544ec96723 Mon Sep 17 00:00:00 2001 From: Reiase Date: Sun, 6 Sep 2026 23:44:34 +0800 Subject: [PATCH 01/11] fix(pchronicle): generate ids for compact JSONL records --- crates/persisting-pchronicle-cli/src/lib.rs | 3 +- .../src/store/compact_jsonl.rs | 33 ++++++++++++++++++- docs/src/en/pchronicle/reference/cli.md | 5 +-- docs/src/en/rfcs/0014-compact-jsonl.md | 13 +++++--- docs/src/zh/pchronicle/reference/cli.md | 5 +-- docs/src/zh/rfcs/0014-compact-jsonl.md | 13 +++++--- 6 files changed, 56 insertions(+), 16 deletions(-) diff --git a/crates/persisting-pchronicle-cli/src/lib.rs b/crates/persisting-pchronicle-cli/src/lib.rs index 7a0ea7b3..e6b0c2b9 100644 --- a/crates/persisting-pchronicle-cli/src/lib.rs +++ b/crates/persisting-pchronicle-cli/src/lib.rs @@ -730,7 +730,8 @@ struct ImportArgs { #[arg(long, value_parser = parse_byte_size, default_value = "256MiB")] max_input_bytes: Option, - /// Compact JSONL mapping. id/timestamp override $.id/$.timestamp; other names add JSONB columns. + /// Compact JSONL mapping. id/timestamp override $.id/$.timestamp; missing or invalid id values + /// use source_filename#line_number; other names add JSONB columns. /// Example: --column id=$.event.id --column model=$.payload.model. #[arg(long = "column", value_name = "NAME=JSON_PATH", action = clap::ArgAction::Append)] columns: Vec, diff --git a/crates/persisting-pchronicle/src/store/compact_jsonl.rs b/crates/persisting-pchronicle/src/store/compact_jsonl.rs index ae112c6c..56140f42 100644 --- a/crates/persisting-pchronicle/src/store/compact_jsonl.rs +++ b/crates/persisting-pchronicle/src/store/compact_jsonl.rs @@ -257,7 +257,15 @@ impl CompactJsonlStore { }) .collect() }; - let ids = required("id", "$.id")?; + let ids = rows + .iter() + .map(|(value, _, file, line)| { + path_value(value, options.path_for("id", "$.id")) + .and_then(scalar_string) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| format!("{file}#{line}")) + }) + .collect::>(); let timestamps = required("timestamp", "$.timestamp")?; let mut unique_ids = std::collections::HashSet::new(); for id in &ids { @@ -725,4 +733,27 @@ mod tests { assert!(error.to_string().contains("requires scalar timestamp")); Ok(()) } + + #[tokio::test] + async fn missing_id_uses_source_line_and_preserves_input() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("events.jsonl"); + let dataset = temp.path().join("data.lance"); + let output = temp.path().join("out"); + let raw = b"{\"timestamp\":1,\"event\":\"start\"}\n{\"timestamp\":2,\"event\":\"end\"}\n"; + fs::write(&input, raw)?; + + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()).await?; + let records = CompactJsonlStore::records(&dataset).await?; + assert_eq!( + records + .iter() + .map(|record| record.id.as_str()) + .collect::>(), + ["events.jsonl#1", "events.jsonl#2"] + ); + CompactJsonlStore::export_path(&dataset, &output).await?; + assert_eq!(fs::read(output.join("events.jsonl"))?, raw); + Ok(()) + } } diff --git a/docs/src/en/pchronicle/reference/cli.md b/docs/src/en/pchronicle/reference/cli.md index c1005932..c5561db6 100644 --- a/docs/src/en/pchronicle/reference/cli.md +++ b/docs/src/en/pchronicle/reference/cli.md @@ -211,8 +211,9 @@ replaced in place. Compact JSONL is a record store, not a trajectory conversion. Either `--input-format compact-jsonl` or `--output-format compact-jsonl` selects it. It recursively reads local `.jsonl` files and requires every non-empty line to -be a JSON object with a unique scalar `id` and scalar `timestamp`; their default -paths are `$.id` and `$.timestamp`. `--column id=PATH` and +be a JSON object with a scalar `timestamp`; its default path is `$.timestamp`. +Missing or invalid `id` values receive a stable `source_filename#line_number` ID; +the default path is `$.id`. `--column id=PATH` and `--column timestamp=PATH` override those paths, while any other `--column NAME=PATH` adds a nullable JSONB projection. Compact import supports local `create` and confirmed `replace`, but not stdin, object-store targets, or diff --git a/docs/src/en/rfcs/0014-compact-jsonl.md b/docs/src/en/rfcs/0014-compact-jsonl.md index 5119d281..d5011d69 100644 --- a/docs/src/en/rfcs/0014-compact-jsonl.md +++ b/docs/src/en/rfcs/0014-compact-jsonl.md @@ -60,7 +60,8 @@ Storyline，也不推断轨迹语义。 1. 不是空行或纯空白行； 2. 是 UTF-8 JSON； 3. 顶层值是 JSON object； -4. 包含可映射为非空标量的 `id` 和 `timestamp`。 +4. 包含可映射为非空标量的 `timestamp`；如果 `id` 缺失或不能映射为非空标量，实现 MUST + 生成稳定的 `source_filename#line_number` ID。 任一条件不满足时，整个 import/sync MUST 失败，不得发布部分 dataset。 @@ -78,7 +79,9 @@ index = "[" 1*DIGIT "]" 示例：`$`、`$.id`、`$.user.id`、`$.messages[0].role`。 不支持 wildcard、slice、filter、递归下降、quoted member、负数 index 或一段中的多个 index。 -映射缺失时，附加列写 Arrow null；`id` 或 `timestamp` 映射缺失时必须拒绝该 record。 +映射缺失时，附加列写 Arrow null；`timestamp` 映射缺失时必须拒绝该 record。`id` 映射缺失 +或无效时，使用该 record 的规范化 source filename 和 1-based line number 生成 ID，格式为 +`source_filename#line_number`。生成的 ID 只写入 Lance `id` 列，不修改 `_raw_` 或 `data`。 ### 列映射 @@ -95,9 +98,9 @@ timestamp=$.timestamp `NAME` MUST 符合上述 `identifier`。映射名 MUST 唯一。`filename`、`data`、`_raw_` 和 `_offload_` 是保留名，MUST NOT 用作用户映射名。 -`id` 和 `timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入；number 使用 -JSON 规范表示写为 UTF-8。null、boolean、array、object 与空 string 均无效。dataset 内的 -`id` MUST 唯一。 +`timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入；number 使用 JSON +规范表示写为 UTF-8。有效的 `id` 来源值也按同样规则写入；其他 `id` 值触发上述生成规则。 +dataset 内的 `id` MUST 唯一。 ## Lance 物理 schema diff --git a/docs/src/zh/pchronicle/reference/cli.md b/docs/src/zh/pchronicle/reference/cli.md index d317417c..edf0aaac 100644 --- a/docs/src/zh/pchronicle/reference/cli.md +++ b/docs/src/zh/pchronicle/reference/cli.md @@ -330,8 +330,9 @@ Storyline Dataset。默认 `create` 模式要求目标不存在。`append` 要 Compact JSONL 是记录存储，不会转换或推断轨迹语义。指定 `--input-format compact-jsonl` 或 `--output-format compact-jsonl` 均会选择该格式。输入必须是本地 -`.jsonl` 文件或目录树；每个非空行必须是 JSON object，并包含唯一的标量 `id` 和标量 -`timestamp`，默认路径分别为 `$.id` 和 `$.timestamp`。`--column id=PATH` 与 +`.jsonl` 文件或目录树；每个非空行必须是 JSON object，并包含标量 `timestamp`，默认路径为 +`$.timestamp`。缺失或无效的 `id` 会获得稳定的 `source_filename#line_number` ID，默认路径为 +`$.id`。`--column id=PATH` 与 `--column timestamp=PATH` 用于覆盖默认路径，其他 `--column NAME=PATH` 会增加 nullable JSONB 投影列。Compact import 支持本地 `create` 和经确认的 `replace`，不支持 stdin、对象存储目标或 `append`。 diff --git a/docs/src/zh/rfcs/0014-compact-jsonl.md b/docs/src/zh/rfcs/0014-compact-jsonl.md index 5119d281..719b7e24 100644 --- a/docs/src/zh/rfcs/0014-compact-jsonl.md +++ b/docs/src/zh/rfcs/0014-compact-jsonl.md @@ -60,7 +60,8 @@ Storyline，也不推断轨迹语义。 1. 不是空行或纯空白行； 2. 是 UTF-8 JSON； 3. 顶层值是 JSON object； -4. 包含可映射为非空标量的 `id` 和 `timestamp`。 +4. 包含可映射为非空标量的 `timestamp`；如果 `id` 缺失或不能映射为非空标量，实现必须 + 生成稳定的 `source_filename#line_number` ID。 任一条件不满足时，整个 import/sync MUST 失败，不得发布部分 dataset。 @@ -78,7 +79,9 @@ index = "[" 1*DIGIT "]" 示例：`$`、`$.id`、`$.user.id`、`$.messages[0].role`。 不支持 wildcard、slice、filter、递归下降、quoted member、负数 index 或一段中的多个 index。 -映射缺失时，附加列写 Arrow null；`id` 或 `timestamp` 映射缺失时必须拒绝该 record。 +映射缺失时，附加列写 Arrow null；`timestamp` 映射缺失时必须拒绝该 record。`id` 映射缺失 +或无效时，使用该 record 的规范化 source filename 和从 1 开始的行号生成 ID，格式为 +`source_filename#line_number`。生成的 ID 只写入 Lance `id` 列，不修改 `_raw_` 或 `data`。 ### 列映射 @@ -95,9 +98,9 @@ timestamp=$.timestamp `NAME` MUST 符合上述 `identifier`。映射名 MUST 唯一。`filename`、`data`、`_raw_` 和 `_offload_` 是保留名，MUST NOT 用作用户映射名。 -`id` 和 `timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入；number 使用 -JSON 规范表示写为 UTF-8。null、boolean、array、object 与空 string 均无效。dataset 内的 -`id` MUST 唯一。 +`timestamp` 的来源值 MUST 是 JSON string 或 number。string 原样写入；number 使用 JSON +规范表示写为 UTF-8。有效的 `id` 来源值也按同样规则写入；其他 `id` 值触发上述生成规则。 +dataset 内的 `id` MUST 唯一。 ## Lance 物理 schema From 707efc778d2345ff3a22cb6c03f075357270de92 Mon Sep 17 00:00:00 2001 From: Reiase Date: Sun, 6 Sep 2026 23:50:36 +0800 Subject: [PATCH 02/11] fix(pchronicle): discover extensionless Lance datasets --- .../src/store/catalog/mod.rs | 1 + .../src/store/catalog/tests.rs | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/crates/persisting-pchronicle/src/store/catalog/mod.rs b/crates/persisting-pchronicle/src/store/catalog/mod.rs index 64ecd350..cf12a608 100644 --- a/crates/persisting-pchronicle/src/store/catalog/mod.rs +++ b/crates/persisting-pchronicle/src/store/catalog/mod.rs @@ -836,6 +836,7 @@ fn is_lance_directory(path: &Path) -> bool { path.extension() .and_then(|extension| extension.to_str()) .is_some_and(|extension| extension.eq_ignore_ascii_case("lance")) + || path.join("_versions").is_dir() } fn path_is_inside_lance_directory(path: &str) -> bool { diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index 89e027bb..f566f607 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -219,6 +219,32 @@ async fn ignores_derived_lance_sidecars_during_discovery() -> Result<()> { Ok(()) } +#[tokio::test] +async fn discovers_extensionless_compact_lance_dataset() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("input.jsonl"); + let compact = temp.path().join("compact"); + fs::write(&input, b"{\"timestamp\":1,\"value\":\"ok\"}\n")?; + crate::storage::CompactJsonlStore::import_path( + &input, + &compact, + &crate::storage::CompactJsonlOptions::default(), + ) + .await?; + + let snapshot = DatasetCatalogSnapshot::discover( + vec![DatasetMount::default(temp.path().to_string_lossy())?], + Some(DEFAULT_DATASET_NAME.into()), + CatalogSnapshotOptions::default(), + ) + .await?; + let sources = &snapshot.datasets()[0].sources; + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].file, "compact"); + assert_eq!(sources[0].format.as_deref(), Some("compact-jsonl/v1")); + Ok(()) +} + #[tokio::test] async fn report_mode_skips_oversized_files_when_querying_all_runs() -> Result<()> { let temp = tempfile::tempdir()?; From 42384d0c238b7cdf8585ed712d9f66e6e414e1a4 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 00:07:13 +0800 Subject: [PATCH 03/11] feat(pchronicle): accept JSON documents in compact import --- crates/persisting-pchronicle-cli/src/lib.rs | 4 +- .../src/store/compact_jsonl.rs | 176 ++++++++++++------ docs/src/en/pchronicle/reference/cli.md | 12 +- docs/src/en/rfcs/0014-compact-jsonl.md | 19 +- docs/src/zh/pchronicle/reference/cli.md | 8 +- docs/src/zh/rfcs/0014-compact-jsonl.md | 19 +- 6 files changed, 156 insertions(+), 82 deletions(-) diff --git a/crates/persisting-pchronicle-cli/src/lib.rs b/crates/persisting-pchronicle-cli/src/lib.rs index e6b0c2b9..231f9705 100644 --- a/crates/persisting-pchronicle-cli/src/lib.rs +++ b/crates/persisting-pchronicle-cli/src/lib.rs @@ -207,7 +207,7 @@ enum Command { /// /// Compact JSONL stores each JSON object as one record without trajectory semantics. /// Select it with either --input-format compact-jsonl or - /// --output-format compact-jsonl; it accepts local .jsonl input and supports + /// --output-format compact-jsonl; it accepts local JSON/JSONL input and supports /// create or replace, not stdin or append. Import(ImportArgs), /// Permanently delete a Dataset directory or object-store prefix. @@ -702,7 +702,7 @@ struct ImportArgs { output: Option, /// Input exchange format. Auto detects run data; compact-jsonl is explicit. - /// Compact JSONL recursively reads local .jsonl files, one object per line. + /// Compact JSONL recursively reads local .json, .jsonl, or .ndjson files. #[arg(short = 'i', long = "input-format", alias = "format", value_enum, default_value_t = ExchangeFormat::Auto)] format: ExchangeFormat, diff --git a/crates/persisting-pchronicle/src/store/compact_jsonl.rs b/crates/persisting-pchronicle/src/store/compact_jsonl.rs index 56140f42..59cc415c 100644 --- a/crates/persisting-pchronicle/src/store/compact_jsonl.rs +++ b/crates/persisting-pchronicle/src/store/compact_jsonl.rs @@ -195,10 +195,10 @@ impl CompactJsonlStore { let input = input.as_ref(); let output = output.as_ref(); options.validate()?; - let files = collect_jsonl(input)?; + let files = collect_json_inputs(input)?; ensure!( !files.is_empty(), - "compact JSONL input contains no .jsonl files" + "compact JSONL input contains no .json, .jsonl, or .ndjson files" ); if output.exists() { fs::remove_dir_all(output) @@ -216,57 +216,74 @@ impl CompactJsonlStore { .to_str() .context("compact JSONL filename is not UTF-8")? .replace('\\', "/"); - let mut reader = BufReader::new(File::open(&file)?); let first_row = rows.len(); - for line_no in 1usize.. { - let mut raw = Vec::new(); - if reader.read_until(b'\n', &mut raw)? == 0 { - break; + if is_json_document(&file) { + let raw = fs::read(&file)?; + let value: Value = serde_json::from_slice(&raw) + .with_context(|| format!("parse compact JSON {relative}"))?; + match value { + Value::Object(_) => rows.push((value, raw, relative.clone(), 1)), + Value::Array(values) => { + for (index, value) in values.into_iter().enumerate() { + ensure!( + value.is_object(), + "compact JSON {relative}:{} must be a JSON object", + index + 1 + ); + let mut raw = serde_json::to_vec(&value)?; + raw.push(b'\n'); + rows.push((value, raw, relative.clone(), index + 1)); + } + } + _ => anyhow::bail!("compact JSON {relative} must be an object or array"), + } + } else { + let mut reader = BufReader::new(File::open(&file)?); + for line_no in 1usize.. { + let mut raw = Vec::new(); + if reader.read_until(b'\n', &mut raw)? == 0 { + break; + } + let mut end = raw.len() - usize::from(raw.ends_with(b"\n")); + end -= usize::from(raw[..end].ends_with(b"\r")); + let json = &raw[..end]; + ensure!( + !json.iter().all(u8::is_ascii_whitespace), + "compact JSONL {}:{} is empty", + relative, + line_no + ); + let value: Value = serde_json::from_slice(json) + .with_context(|| format!("parse compact JSONL {relative}:{line_no}"))?; + ensure!( + value.is_object(), + "compact JSONL {relative}:{line_no} must be a JSON object" + ); + rows.push((value, raw, relative.clone(), line_no)); } - let mut end = raw.len() - usize::from(raw.ends_with(b"\n")); - end -= usize::from(raw[..end].ends_with(b"\r")); - let json = &raw[..end]; - ensure!( - !json.iter().all(u8::is_ascii_whitespace), - "compact JSONL {}:{} is empty", - relative, - line_no - ); - let value: Value = serde_json::from_slice(json) - .with_context(|| format!("parse compact JSONL {relative}:{line_no}"))?; - ensure!( - value.is_object(), - "compact JSONL {relative}:{line_no} must be a JSON object" - ); - rows.push((value, raw, relative.clone(), line_no)); } ensure!(rows.len() > first_row, "compact JSONL {relative} is empty"); } let schema = schema(options)?; let mut arrays: Vec> = Vec::new(); - let required = |name: &str, default| -> Result> { - let path = options.path_for(name, default); - rows.iter() - .map(|(value, _, file, line)| { - path_value(value, path) - .and_then(scalar_string) - .filter(|value| !value.is_empty()) - .with_context(|| { - format!("compact JSONL {file}:{line} requires scalar {name} at {path}") - }) - }) - .collect() - }; let ids = rows .iter() .map(|(value, _, file, line)| { path_value(value, options.path_for("id", "$.id")) .and_then(scalar_string) .filter(|value| !value.is_empty()) - .unwrap_or_else(|| format!("{file}#{line}")) + .unwrap_or_else(|| generated_record_key(file, *line)) + }) + .collect::>(); + let timestamps = rows + .iter() + .map(|(value, _, file, line)| { + path_value(value, options.path_for("timestamp", "$.timestamp")) + .and_then(scalar_string) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| generated_record_key(file, *line)) }) .collect::>(); - let timestamps = required("timestamp", "$.timestamp")?; let mut unique_ids = std::collections::HashSet::new(); for id in &ids { ensure!(unique_ids.insert(id), "duplicate compact JSONL id '{id}'"); @@ -538,7 +555,7 @@ fn encode_json_bytes(value: &str) -> Result> { encode_json(value).map_err(|error| anyhow!("encode compact JSON value: {error}")) } -fn collect_jsonl(input: &Path) -> Result> { +fn collect_json_inputs(input: &Path) -> Result> { let metadata = fs::symlink_metadata(input) .with_context(|| format!("inspect compact JSONL input {}", input.display()))?; ensure!( @@ -550,8 +567,11 @@ fn collect_jsonl(input: &Path) -> Result> { input .extension() .and_then(|x| x.to_str()) - .is_some_and(|x| x.eq_ignore_ascii_case("jsonl")), - "compact JSONL input must be .jsonl" + .is_some_and(|x| matches!( + x.to_ascii_lowercase().as_str(), + "json" | "jsonl" | "ndjson" + )), + "compact JSONL input must be .json, .jsonl, or .ndjson" ); return Ok(vec![input.to_path_buf()]); } @@ -571,11 +591,7 @@ fn collect_jsonl(input: &Path) -> Result> { let p = entry.path(); if file_type.is_dir() { stack.push(p); - } else if file_type.is_file() - && p.extension() - .and_then(|x| x.to_str()) - .is_some_and(|x| x.eq_ignore_ascii_case("jsonl")) - { + } else if file_type.is_file() && is_json_input(&p) { out.push(p); } } @@ -584,6 +600,27 @@ fn collect_jsonl(input: &Path) -> Result> { Ok(out) } +fn is_json_input(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + matches!( + extension.to_ascii_lowercase().as_str(), + "json" | "jsonl" | "ndjson" + ) + }) +} + +fn is_json_document(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("json")) +} + +fn generated_record_key(file: &str, line: usize) -> String { + format!("{file}#{line}") +} + fn valid_path(path: &str) -> bool { if path == "$" { return true; @@ -719,18 +756,47 @@ mod tests { } #[tokio::test] - async fn missing_required_timestamp_rejects_the_snapshot() -> Result<()> { + async fn missing_base_columns_use_source_line_values() -> Result<()> { let temp = tempfile::tempdir()?; let input = temp.path().join("input.jsonl"); fs::write(&input, b"{\"id\":\"only\"}\n")?; - let error = CompactJsonlStore::import_path( - &input, - temp.path().join("data.lance"), - &CompactJsonlOptions::default(), - ) - .await - .unwrap_err(); - assert!(error.to_string().contains("requires scalar timestamp")); + let dataset = temp.path().join("data.lance"); + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()).await?; + let record = &CompactJsonlStore::records(&dataset).await?[0]; + assert_eq!(record.id, "input.jsonl#1"); + assert_eq!(record.timestamp, "input.jsonl#1"); + Ok(()) + } + + #[tokio::test] + async fn json_documents_and_arrays_are_imported() -> Result<()> { + let temp = tempfile::tempdir()?; + let input = temp.path().join("input"); + let dataset = temp.path().join("data.lance"); + let output = temp.path().join("out"); + fs::create_dir_all(&input)?; + fs::write(input.join("object.json"), b"{\"value\":1}")?; + fs::write(input.join("array.json"), b"[{\"value\":2},{\"value\":3}]")?; + + assert_eq!( + CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()) + .await?, + 3 + ); + let records = CompactJsonlStore::records(&dataset).await?; + assert_eq!( + records + .iter() + .map(|record| record.id.as_str()) + .collect::>(), + ["array.json#1", "array.json#2", "object.json#1"] + ); + CompactJsonlStore::export_path(&dataset, &output).await?; + assert_eq!( + fs::read(output.join("array.json"))?, + b"{\"value\":2}\n{\"value\":3}\n" + ); + assert_eq!(fs::read(output.join("object.json"))?, b"{\"value\":1}"); Ok(()) } diff --git a/docs/src/en/pchronicle/reference/cli.md b/docs/src/en/pchronicle/reference/cli.md index c5561db6..06af44d3 100644 --- a/docs/src/en/pchronicle/reference/cli.md +++ b/docs/src/en/pchronicle/reference/cli.md @@ -210,10 +210,11 @@ replaced in place. Compact JSONL is a record store, not a trajectory conversion. Either `--input-format compact-jsonl` or `--output-format compact-jsonl` selects it. -It recursively reads local `.jsonl` files and requires every non-empty line to -be a JSON object with a scalar `timestamp`; its default path is `$.timestamp`. -Missing or invalid `id` values receive a stable `source_filename#line_number` ID; -the default path is `$.id`. `--column id=PATH` and +It recursively reads local `.json`, `.jsonl`, and `.ndjson` files. JSON objects +and arrays of objects are accepted; array elements become individual records. +Missing or invalid `id` and `timestamp` values receive stable +`source_filename#line_number` values. The default paths are `$.id` and +`$.timestamp`. `--column id=PATH` and `--column timestamp=PATH` override those paths, while any other `--column NAME=PATH` adds a nullable JSONB projection. Compact import supports local `create` and confirmed `replace`, but not stdin, object-store targets, or @@ -236,7 +237,8 @@ the set and retry with bounded exponential backoff. Use `--once` for one initial batch and exit. The two destinations must be local directories outside the source directory. -With `--input-format compact-jsonl`, the source must be a local `.jsonl` tree +With `--input-format compact-jsonl`, the source must be a local `.json`, `.jsonl`, +or `.ndjson` tree and the same `--column` rules as compact import apply. Every successful batch rescans the whole tree and atomically replaces the compact Lance snapshot at `--convert`, so additions, changes, and deletions are reflected without diff --git a/docs/src/en/rfcs/0014-compact-jsonl.md b/docs/src/en/rfcs/0014-compact-jsonl.md index d5011d69..e0713d92 100644 --- a/docs/src/en/rfcs/0014-compact-jsonl.md +++ b/docs/src/en/rfcs/0014-compact-jsonl.md @@ -51,17 +51,20 @@ Storyline，也不推断轨迹语义。 ### 文件集合 -显式文件输入 MUST 是扩展名大小写不敏感的 `.jsonl` 普通文件。目录输入 MUST 递归选择 -`.jsonl` 普通文件，并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 +显式文件输入 MUST 是扩展名大小写不敏感的 `.json`、`.jsonl` 或 `.ndjson` 普通文件。目录输入 +MUST 递归选择这些普通文件，并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 字节序处理文件；单个文件内部 MUST 按原始行顺序处理。 -每个被选择的文件 MUST 至少包含一条 record。每一物理行 MUST： +`.jsonl` 和 `.ndjson` 文件中的每一物理行 MUST： 1. 不是空行或纯空白行； 2. 是 UTF-8 JSON； 3. 顶层值是 JSON object； -4. 包含可映射为非空标量的 `timestamp`；如果 `id` 缺失或不能映射为非空标量，实现 MUST - 生成稳定的 `source_filename#line_number` ID。 +4. 包含可映射为非空标量的 `timestamp`；缺失或无效的 `id` 和 `timestamp` 使用稳定的 + `source_filename#line_number` 值生成。 + +`.json` 文件 MUST 是一个 object 或 object 数组。object 数组的每个元素成为一条 record，数组 +元素按顺序使用 1-based index 作为 line number；导出时这类文件以 JSONL 记录写回。 任一条件不满足时，整个 import/sync MUST 失败，不得发布部分 dataset。 @@ -79,9 +82,9 @@ index = "[" 1*DIGIT "]" 示例：`$`、`$.id`、`$.user.id`、`$.messages[0].role`。 不支持 wildcard、slice、filter、递归下降、quoted member、负数 index 或一段中的多个 index。 -映射缺失时，附加列写 Arrow null；`timestamp` 映射缺失时必须拒绝该 record。`id` 映射缺失 -或无效时，使用该 record 的规范化 source filename 和 1-based line number 生成 ID，格式为 -`source_filename#line_number`。生成的 ID 只写入 Lance `id` 列，不修改 `_raw_` 或 `data`。 +映射缺失时，附加列写 Arrow null；`id` 或 `timestamp` 映射缺失或无效时，使用该 record 的 +规范化 source filename 和 1-based line number 生成值，格式为 `source_filename#line_number`。 +生成的值只写入 Lance 基础列，不修改 `_raw_` 或 `data`。 ### 列映射 diff --git a/docs/src/zh/pchronicle/reference/cli.md b/docs/src/zh/pchronicle/reference/cli.md index edf0aaac..04f80185 100644 --- a/docs/src/zh/pchronicle/reference/cli.md +++ b/docs/src/zh/pchronicle/reference/cli.md @@ -330,9 +330,9 @@ Storyline Dataset。默认 `create` 模式要求目标不存在。`append` 要 Compact JSONL 是记录存储，不会转换或推断轨迹语义。指定 `--input-format compact-jsonl` 或 `--output-format compact-jsonl` 均会选择该格式。输入必须是本地 -`.jsonl` 文件或目录树；每个非空行必须是 JSON object，并包含标量 `timestamp`，默认路径为 -`$.timestamp`。缺失或无效的 `id` 会获得稳定的 `source_filename#line_number` ID，默认路径为 -`$.id`。`--column id=PATH` 与 +`.json`、`.jsonl` 或 `.ndjson` 文件或目录树；JSON object 和 object 数组都会被接受，数组元素 +会成为独立 record。缺失或无效的 `id` 和 `timestamp` 会获得稳定的 +`source_filename#line_number` 值，默认路径分别为 `$.id` 和 `$.timestamp`。`--column id=PATH` 与 `--column timestamp=PATH` 用于覆盖默认路径，其他 `--column NAME=PATH` 会增加 nullable JSONB 投影列。Compact import 支持本地 `create` 和经确认的 `replace`，不支持 stdin、对象存储目标或 `append`。 @@ -351,7 +351,7 @@ pchronicle sync --from DIRECTORY --to DIRECTORY --convert DIRECTORY 变更并指数退避重试。`--once` 只执行一次初始批次后退出。当前目标必须是本地目录，两个目标 必须位于源目录之外。 -指定 `--input-format compact-jsonl` 时，源目录必须是本地 `.jsonl` 目录树，列映射规则与 Compact +指定 `--input-format compact-jsonl` 时，源目录必须是本地 `.json`、`.jsonl` 或 `.ndjson` 目录树，列映射规则与 Compact import 相同。每个成功批次都会重新扫描整个目录，并原子替换 `--convert` 指向的 Compact Lance 快照，因此新增、修改和删除都会反映在下一快照中，但不提供行级增量更新。此模式仍要求传入 `--to` 作为兼容参数，但不会写入该路径。 diff --git a/docs/src/zh/rfcs/0014-compact-jsonl.md b/docs/src/zh/rfcs/0014-compact-jsonl.md index 719b7e24..0a5989bf 100644 --- a/docs/src/zh/rfcs/0014-compact-jsonl.md +++ b/docs/src/zh/rfcs/0014-compact-jsonl.md @@ -51,17 +51,20 @@ Storyline，也不推断轨迹语义。 ### 文件集合 -显式文件输入 MUST 是扩展名大小写不敏感的 `.jsonl` 普通文件。目录输入 MUST 递归选择 -`.jsonl` 普通文件，并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 +显式文件输入 MUST 是扩展名大小写不敏感的 `.json`、`.jsonl` 或 `.ndjson` 普通文件。目录输入 +MUST 递归选择这些普通文件，并 MUST 忽略符号链接和其他扩展名。实现 MUST 按规范化相对文件名的 字节序处理文件；单个文件内部 MUST 按原始行顺序处理。 -每个被选择的文件 MUST 至少包含一条 record。每一物理行 MUST： +`.jsonl` 和 `.ndjson` 文件中的每一物理行 MUST： 1. 不是空行或纯空白行； 2. 是 UTF-8 JSON； 3. 顶层值是 JSON object； -4. 包含可映射为非空标量的 `timestamp`；如果 `id` 缺失或不能映射为非空标量，实现必须 - 生成稳定的 `source_filename#line_number` ID。 +4. 包含可映射为非空标量的 `timestamp`；缺失或无效的 `id` 和 `timestamp` 使用稳定的 + `source_filename#line_number` 值生成。 + +`.json` 文件必须是一个 object 或 object 数组。object 数组的每个元素成为一条 record，数组 +元素按顺序使用从 1 开始的索引作为行号；导出时这类文件以 JSONL 记录写回。 任一条件不满足时，整个 import/sync MUST 失败，不得发布部分 dataset。 @@ -79,9 +82,9 @@ index = "[" 1*DIGIT "]" 示例：`$`、`$.id`、`$.user.id`、`$.messages[0].role`。 不支持 wildcard、slice、filter、递归下降、quoted member、负数 index 或一段中的多个 index。 -映射缺失时，附加列写 Arrow null；`timestamp` 映射缺失时必须拒绝该 record。`id` 映射缺失 -或无效时，使用该 record 的规范化 source filename 和从 1 开始的行号生成 ID，格式为 -`source_filename#line_number`。生成的 ID 只写入 Lance `id` 列，不修改 `_raw_` 或 `data`。 +映射缺失时，附加列写 Arrow null；`id` 或 `timestamp` 映射缺失或无效时，使用该 record 的 +规范化 source filename 和从 1 开始的行号生成值，格式为 `source_filename#line_number`。 +生成的值只写入 Lance 基础列，不修改 `_raw_` 或 `data`。 ### 列映射 From 3c53b64b1e718a89f8770a3f20372781747a8036 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 00:20:06 +0800 Subject: [PATCH 04/11] fix(pchronicle): keep JSON arrays as single records --- .../src/store/compact_jsonl.rs | 29 ++++++------------- docs/src/en/pchronicle/reference/cli.md | 2 +- docs/src/en/rfcs/0014-compact-jsonl.md | 4 +-- docs/src/zh/pchronicle/reference/cli.md | 4 +-- docs/src/zh/rfcs/0014-compact-jsonl.md | 4 +-- 5 files changed, 16 insertions(+), 27 deletions(-) diff --git a/crates/persisting-pchronicle/src/store/compact_jsonl.rs b/crates/persisting-pchronicle/src/store/compact_jsonl.rs index 59cc415c..04a99790 100644 --- a/crates/persisting-pchronicle/src/store/compact_jsonl.rs +++ b/crates/persisting-pchronicle/src/store/compact_jsonl.rs @@ -221,22 +221,11 @@ impl CompactJsonlStore { let raw = fs::read(&file)?; let value: Value = serde_json::from_slice(&raw) .with_context(|| format!("parse compact JSON {relative}"))?; - match value { - Value::Object(_) => rows.push((value, raw, relative.clone(), 1)), - Value::Array(values) => { - for (index, value) in values.into_iter().enumerate() { - ensure!( - value.is_object(), - "compact JSON {relative}:{} must be a JSON object", - index + 1 - ); - let mut raw = serde_json::to_vec(&value)?; - raw.push(b'\n'); - rows.push((value, raw, relative.clone(), index + 1)); - } - } - _ => anyhow::bail!("compact JSON {relative} must be an object or array"), - } + ensure!( + matches!(value, Value::Object(_) | Value::Array(_)), + "compact JSON {relative} must be an object or array" + ); + rows.push((value, raw, relative.clone(), 1)); } else { let mut reader = BufReader::new(File::open(&file)?); for line_no in 1usize.. { @@ -769,7 +758,7 @@ mod tests { } #[tokio::test] - async fn json_documents_and_arrays_are_imported() -> Result<()> { + async fn json_documents_and_arrays_are_imported_as_records() -> Result<()> { let temp = tempfile::tempdir()?; let input = temp.path().join("input"); let dataset = temp.path().join("data.lance"); @@ -781,7 +770,7 @@ mod tests { assert_eq!( CompactJsonlStore::import_path(&input, &dataset, &CompactJsonlOptions::default()) .await?, - 3 + 2 ); let records = CompactJsonlStore::records(&dataset).await?; assert_eq!( @@ -789,12 +778,12 @@ mod tests { .iter() .map(|record| record.id.as_str()) .collect::>(), - ["array.json#1", "array.json#2", "object.json#1"] + ["array.json#1", "object.json#1"] ); CompactJsonlStore::export_path(&dataset, &output).await?; assert_eq!( fs::read(output.join("array.json"))?, - b"{\"value\":2}\n{\"value\":3}\n" + b"[{\"value\":2},{\"value\":3}]" ); assert_eq!(fs::read(output.join("object.json"))?, b"{\"value\":1}"); Ok(()) diff --git a/docs/src/en/pchronicle/reference/cli.md b/docs/src/en/pchronicle/reference/cli.md index 06af44d3..df5b3d73 100644 --- a/docs/src/en/pchronicle/reference/cli.md +++ b/docs/src/en/pchronicle/reference/cli.md @@ -211,7 +211,7 @@ replaced in place. Compact JSONL is a record store, not a trajectory conversion. Either `--input-format compact-jsonl` or `--output-format compact-jsonl` selects it. It recursively reads local `.json`, `.jsonl`, and `.ndjson` files. JSON objects -and arrays of objects are accepted; array elements become individual records. +and arrays are accepted; each JSON document becomes one record, with arrays kept intact. Missing or invalid `id` and `timestamp` values receive stable `source_filename#line_number` values. The default paths are `$.id` and `$.timestamp`. `--column id=PATH` and diff --git a/docs/src/en/rfcs/0014-compact-jsonl.md b/docs/src/en/rfcs/0014-compact-jsonl.md index e0713d92..9925d063 100644 --- a/docs/src/en/rfcs/0014-compact-jsonl.md +++ b/docs/src/en/rfcs/0014-compact-jsonl.md @@ -63,8 +63,8 @@ MUST 递归选择这些普通文件，并 MUST 忽略符号链接和其他扩展 4. 包含可映射为非空标量的 `timestamp`；缺失或无效的 `id` 和 `timestamp` 使用稳定的 `source_filename#line_number` 值生成。 -`.json` 文件 MUST 是一个 object 或 object 数组。object 数组的每个元素成为一条 record，数组 -元素按顺序使用 1-based index 作为 line number；导出时这类文件以 JSONL 记录写回。 +`.json` 文件 MUST 是一个 object 或 array，并作为一条 record 导入；数组保持为 record 的 JSON +值，不会按元素拆分。导出时保留该 JSON document。 任一条件不满足时，整个 import/sync MUST 失败，不得发布部分 dataset。 diff --git a/docs/src/zh/pchronicle/reference/cli.md b/docs/src/zh/pchronicle/reference/cli.md index 04f80185..daacab2c 100644 --- a/docs/src/zh/pchronicle/reference/cli.md +++ b/docs/src/zh/pchronicle/reference/cli.md @@ -330,8 +330,8 @@ Storyline Dataset。默认 `create` 模式要求目标不存在。`append` 要 Compact JSONL 是记录存储，不会转换或推断轨迹语义。指定 `--input-format compact-jsonl` 或 `--output-format compact-jsonl` 均会选择该格式。输入必须是本地 -`.json`、`.jsonl` 或 `.ndjson` 文件或目录树；JSON object 和 object 数组都会被接受，数组元素 -会成为独立 record。缺失或无效的 `id` 和 `timestamp` 会获得稳定的 +`.json`、`.jsonl` 或 `.ndjson` 文件或目录树；JSON object 和 array 都会被接受，每个 JSON 文档 +会成为一条 record，数组会完整保留。缺失或无效的 `id` 和 `timestamp` 会获得稳定的 `source_filename#line_number` 值，默认路径分别为 `$.id` 和 `$.timestamp`。`--column id=PATH` 与 `--column timestamp=PATH` 用于覆盖默认路径，其他 `--column NAME=PATH` 会增加 nullable JSONB 投影列。Compact import 支持本地 `create` 和经确认的 `replace`，不支持 stdin、对象存储目标或 diff --git a/docs/src/zh/rfcs/0014-compact-jsonl.md b/docs/src/zh/rfcs/0014-compact-jsonl.md index 0a5989bf..7bf073bb 100644 --- a/docs/src/zh/rfcs/0014-compact-jsonl.md +++ b/docs/src/zh/rfcs/0014-compact-jsonl.md @@ -63,8 +63,8 @@ MUST 递归选择这些普通文件，并 MUST 忽略符号链接和其他扩展 4. 包含可映射为非空标量的 `timestamp`；缺失或无效的 `id` 和 `timestamp` 使用稳定的 `source_filename#line_number` 值生成。 -`.json` 文件必须是一个 object 或 object 数组。object 数组的每个元素成为一条 record，数组 -元素按顺序使用从 1 开始的索引作为行号；导出时这类文件以 JSONL 记录写回。 +`.json` 文件必须是一个 object 或 array，并作为一条 record 导入；数组会作为 record 的 JSON +值完整保留，不会按元素拆分。导出时保留该 JSON document。 任一条件不满足时，整个 import/sync MUST 失败，不得发布部分 dataset。 From 8830032541e4de7e8544d95198e8f6715e1995bb Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 00:35:57 +0800 Subject: [PATCH 05/11] fix(pchronicle-web): collapse nested compact JSON --- pchronicle-web/assets/components.css | 5 +++++ pchronicle-web/src/json_value.rs | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pchronicle-web/assets/components.css b/pchronicle-web/assets/components.css index e3935ff4..fe5d123b 100644 --- a/pchronicle-web/assets/components.css +++ b/pchronicle-web/assets/components.css @@ -474,18 +474,23 @@ } .pc2-json-scalar { + display: block; + max-width: 100%; + line-height: 1.45; white-space: pre-wrap; word-break: break-word; color: #344054; } .pc2-json-tree { + min-width: 0; display: flex; flex-direction: column; gap: 2px; } .pc2-json-node { + min-width: 0; border-left: 1px solid #e4e7ec; padding-left: 8px; } diff --git a/pchronicle-web/src/json_value.rs b/pchronicle-web/src/json_value.rs index fc3504ca..dd3aa4e1 100644 --- a/pchronicle-web/src/json_value.rs +++ b/pchronicle-web/src/json_value.rs @@ -183,7 +183,7 @@ fn JsonTree(value: Value, default_open: bool) -> Element { Value::Array(items) => rsx! { div { class: "pc2-json-tree", for (index, child) in items.into_iter().enumerate() { - JsonTreeNode { key: "{index}", label: format!("[{index}]"), value: child, default_open } + JsonTreeNode { key: "{index}", label: format!("[{index}]"), value: child, default_open: false } } } }, @@ -200,7 +200,7 @@ fn JsonTreeNode(label: String, value: Value, default_open: bool) -> Element { rsx! { details { class: "pc2-json-node", open: default_open, summary { span { class: "pc2-json-key", "{label}" } span { class: "pc2-json-size", "{summary}" } } - JsonValue { value, default_open } + JsonValue { value, default_open: false } } } } From 7eecbe8d1cecfc71c9f5b7f4d97f24d8d24ecc0e Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 00:42:00 +0800 Subject: [PATCH 06/11] fix(pchronicle-web): compact JSON tree leaves --- pchronicle-web/assets/components.css | 26 +++++++++++++++++++++++++- pchronicle-web/src/json_value.rs | 23 ++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/pchronicle-web/assets/components.css b/pchronicle-web/assets/components.css index fe5d123b..d61357a2 100644 --- a/pchronicle-web/assets/components.css +++ b/pchronicle-web/assets/components.css @@ -486,23 +486,42 @@ min-width: 0; display: flex; flex-direction: column; - gap: 2px; + gap: 3px; } .pc2-json-node { min-width: 0; + margin-left: 8px; border-left: 1px solid #e4e7ec; padding-left: 8px; } +.pc2-json-leaf { + display: grid; + grid-template-columns: minmax(120px, 220px) 58px minmax(0, 1fr); + align-items: baseline; + gap: 8px; + min-width: 0; + padding: 3px 8px; + border-bottom: 1px solid #f2f4f7; +} + .pc2-json-node > summary { display: flex; align-items: center; gap: 8px; + min-height: 24px; + padding: 3px 8px; + border-radius: 4px; + background: #f8fafc; cursor: pointer; list-style: none; } +.pc2-json-node > summary:hover { + background: #eff6ff; +} + .pc2-json-node > summary::-webkit-details-marker { display: none; } @@ -518,6 +537,11 @@ font-size: 9px; } +.pc2-json-type { + color: #98a2b3; + font: 9px ui-monospace, SFMono-Regular, Menlo, monospace; +} + .pc2-json-node > .pc2-json-table, .pc2-json-node > .pc2-json-tree, .pc2-json-node > .pc2-json-scroll, diff --git a/pchronicle-web/src/json_value.rs b/pchronicle-web/src/json_value.rs index dd3aa4e1..e2a6644e 100644 --- a/pchronicle-web/src/json_value.rs +++ b/pchronicle-web/src/json_value.rs @@ -95,6 +95,17 @@ fn scalar_text(value: &Value) -> String { } } +fn json_type(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + #[component] pub fn JsonValue(value: Value, #[props(default = false)] default_open: bool) -> Element { let peeled = peel_json(&value); @@ -196,10 +207,20 @@ fn JsonTree(value: Value, default_open: bool) -> Element { #[component] fn JsonTreeNode(label: String, value: Value, default_open: bool) -> Element { + let peeled = peel_json(&value); + if is_scalar(&peeled) { + return rsx! { + div { class: "pc2-json-leaf", + span { class: "pc2-json-key", "{label}" } + span { class: "pc2-json-type", "{json_type(&peeled)}" } + span { class: "pc2-json-scalar", "{scalar_text(&peeled)}" } + } + }; + } let summary = json_summary(&value); rsx! { details { class: "pc2-json-node", open: default_open, - summary { span { class: "pc2-json-key", "{label}" } span { class: "pc2-json-size", "{summary}" } } + summary { span { class: "pc2-json-key", "{label}" } span { class: "pc2-json-type", "{json_type(&peeled)}" } span { class: "pc2-json-size", "{summary}" } } JsonValue { value, default_open: false } } } From a470d9e2ccdef3cba7965d9f8f9bd30f7975dcef Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 00:51:08 +0800 Subject: [PATCH 07/11] fix(pchronicle-web): render compact JSON like JSONView --- pchronicle-web/assets/components.css | 76 +++++++++++++++++++++++----- pchronicle-web/src/json_value.rs | 20 +++++--- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/pchronicle-web/assets/components.css b/pchronicle-web/assets/components.css index d61357a2..41080390 100644 --- a/pchronicle-web/assets/components.css +++ b/pchronicle-web/assets/components.css @@ -486,24 +486,40 @@ min-width: 0; display: flex; flex-direction: column; - gap: 3px; + gap: 0; + padding-left: 14px; + font: 12px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; +} + +.pc2-json-object::before { + content: "{"; +} + +.pc2-json-object::after { + content: "}"; +} + +.pc2-json-array::before { + content: "["; +} + +.pc2-json-array::after { + content: "]"; } .pc2-json-node { min-width: 0; - margin-left: 8px; - border-left: 1px solid #e4e7ec; - padding-left: 8px; + margin-left: 4px; + border-left: 1px dotted #cbd5e1; + padding-left: 12px; } .pc2-json-leaf { - display: grid; - grid-template-columns: minmax(120px, 220px) 58px minmax(0, 1fr); + display: flex; align-items: baseline; gap: 8px; min-width: 0; - padding: 3px 8px; - border-bottom: 1px solid #f2f4f7; + padding: 1px 4px; } .pc2-json-node > summary { @@ -511,15 +527,25 @@ align-items: center; gap: 8px; min-height: 24px; - padding: 3px 8px; - border-radius: 4px; - background: #f8fafc; + padding: 1px 4px; cursor: pointer; list-style: none; } +.pc2-json-node > summary::before { + width: 11px; + color: #667085; + content: "+"; + font-weight: 700; + text-align: center; +} + +.pc2-json-node[open] > summary::before { + content: "−"; +} + .pc2-json-node > summary:hover { - background: #eff6ff; + background: #f1f5f9; } .pc2-json-node > summary::-webkit-details-marker { @@ -537,6 +563,32 @@ font-size: 9px; } +.pc2-json-punctuation { + color: #667085; +} + +.pc2-json-value { + min-width: 0; + white-space: pre-wrap; + word-break: break-word; +} + +.pc2-json-value.string { + color: #067647; +} + +.pc2-json-value.number { + color: #175cd3; +} + +.pc2-json-value.boolean { + color: #b54708; +} + +.pc2-json-value.null { + color: #98a2b3; +} + .pc2-json-type { color: #98a2b3; font: 9px ui-monospace, SFMono-Regular, Menlo, monospace; diff --git a/pchronicle-web/src/json_value.rs b/pchronicle-web/src/json_value.rs index e2a6644e..60f334dd 100644 --- a/pchronicle-web/src/json_value.rs +++ b/pchronicle-web/src/json_value.rs @@ -95,6 +95,10 @@ fn scalar_text(value: &Value) -> String { } } +fn json_literal(value: &Value) -> String { + serde_json::to_string(value).unwrap_or_else(|_| scalar_text(value)) +} + fn json_type(value: &Value) -> &'static str { match value { Value::Null => "null", @@ -185,16 +189,16 @@ fn JsonTree(value: Value, default_open: bool) -> Element { } }, Value::Object(map) => rsx! { - div { class: "pc2-json-tree", + div { class: "pc2-json-tree pc2-json-object", for (key, child) in map { JsonTreeNode { key: "{key}", label: key, value: child, default_open } } } }, Value::Array(items) => rsx! { - div { class: "pc2-json-tree", + div { class: "pc2-json-tree pc2-json-array", for (index, child) in items.into_iter().enumerate() { - JsonTreeNode { key: "{index}", label: format!("[{index}]"), value: child, default_open: false } + JsonTreeNode { key: "{index}", label: String::new(), value: child, default_open: false } } } }, @@ -211,16 +215,18 @@ fn JsonTreeNode(label: String, value: Value, default_open: bool) -> Element { if is_scalar(&peeled) { return rsx! { div { class: "pc2-json-leaf", - span { class: "pc2-json-key", "{label}" } - span { class: "pc2-json-type", "{json_type(&peeled)}" } - span { class: "pc2-json-scalar", "{scalar_text(&peeled)}" } + if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } + span { class: "pc2-json-value {json_type(&peeled)}", "{json_literal(&peeled)}" } } }; } let summary = json_summary(&value); rsx! { details { class: "pc2-json-node", open: default_open, - summary { span { class: "pc2-json-key", "{label}" } span { class: "pc2-json-type", "{json_type(&peeled)}" } span { class: "pc2-json-size", "{summary}" } } + summary { + if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } + span { class: "pc2-json-size", "{summary}" } + } JsonValue { value, default_open: false } } } From 75fc4a5154c22d40a08abdc9c3c2bd3acc2a6543 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 01:02:21 +0800 Subject: [PATCH 08/11] fix(pchronicle-web): collapse long JSON values --- pchronicle-web/assets/components.css | 34 ++++++++++++++++++++++ pchronicle-web/src/json_value.rs | 42 +++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/pchronicle-web/assets/components.css b/pchronicle-web/assets/components.css index 41080390..3f4f1ceb 100644 --- a/pchronicle-web/assets/components.css +++ b/pchronicle-web/assets/components.css @@ -589,6 +589,40 @@ color: #98a2b3; } +.pc2-json-long-value { + min-width: 0; + flex: 1; +} + +.pc2-json-long-value > summary { + overflow: hidden; + cursor: pointer; + list-style: none; + text-overflow: ellipsis; + white-space: nowrap; +} + +.pc2-json-long-value > summary::-webkit-details-marker { + display: none; +} + +.pc2-json-long-value > summary::before { + margin-right: 4px; + color: #98a2b3; + content: "…"; +} + +.pc2-json-expanded-value { + max-height: 280px; + margin-top: 4px; + overflow: auto; + padding: 6px 8px; + border-left: 2px solid #d0d5dd; + background: #f8fafc; + white-space: pre-wrap; + word-break: break-word; +} + .pc2-json-type { color: #98a2b3; font: 9px ui-monospace, SFMono-Regular, Menlo, monospace; diff --git a/pchronicle-web/src/json_value.rs b/pchronicle-web/src/json_value.rs index 60f334dd..7ddfe5f5 100644 --- a/pchronicle-web/src/json_value.rs +++ b/pchronicle-web/src/json_value.rs @@ -1,6 +1,8 @@ use dioxus::prelude::*; use serde_json::Value; +const JSON_VALUE_PREVIEW_LIMIT: usize = 240; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum JsonShape { Scalar, @@ -99,6 +101,19 @@ fn json_literal(value: &Value) -> String { serde_json::to_string(value).unwrap_or_else(|_| scalar_text(value)) } +fn json_preview(value: &str) -> String { + let mut chars = value.chars(); + let preview = chars + .by_ref() + .take(JSON_VALUE_PREVIEW_LIMIT) + .collect::(); + if chars.next().is_some() { + format!("{preview}…") + } else { + preview + } +} + fn json_type(value: &Value) -> &'static str { match value { Value::Null => "null", @@ -124,6 +139,22 @@ pub fn JsonValue(value: Value, #[props(default = false)] default_open: bool) -> } } +#[component] +fn JsonScalar(value: Value) -> Element { + let literal = json_literal(&value); + let kind = json_type(&value); + if literal.chars().count() > JSON_VALUE_PREVIEW_LIMIT { + rsx! { + details { class: "pc2-json-long-value", + summary { class: "pc2-json-value {kind}", "{json_preview(&literal)}" } + div { class: "pc2-json-expanded-value {kind}", "{literal}" } + } + } + } else { + rsx! { span { class: "pc2-json-value {kind}", "{literal}" } } + } +} + #[component] fn JsonKvTable(value: Value, default_open: bool) -> Element { let map = match value { @@ -216,7 +247,7 @@ fn JsonTreeNode(label: String, value: Value, default_open: bool) -> Element { return rsx! { div { class: "pc2-json-leaf", if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } - span { class: "pc2-json-value {json_type(&peeled)}", "{json_literal(&peeled)}" } + JsonScalar { value: peeled } } }; } @@ -306,4 +337,13 @@ mod tests { assert_eq!(json_summary(&json!(1)), "number"); assert_eq!(json_summary(&json!(null)), "null"); } + + #[test] + fn long_json_values_get_single_line_previews() { + let value = "x".repeat(JSON_VALUE_PREVIEW_LIMIT + 1); + let preview = json_preview(&value); + assert_eq!(preview.chars().count(), JSON_VALUE_PREVIEW_LIMIT + 1); + assert!(preview.ends_with('…')); + assert_eq!(json_preview("short"), "short"); + } } From a806ad0131ff6099f7e81d3550a31c9c05a1d56b Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 06:39:16 +0800 Subject: [PATCH 09/11] test(pchronicle): isolate compact catalog fixture --- crates/persisting-pchronicle/src/store/catalog/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index f566f607..e4ad49c3 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -231,6 +231,7 @@ async fn discovers_extensionless_compact_lance_dataset() -> Result<()> { &crate::storage::CompactJsonlOptions::default(), ) .await?; + fs::remove_file(input)?; let snapshot = DatasetCatalogSnapshot::discover( vec![DatasetMount::default(temp.path().to_string_lossy())?], From f0f6f41ae55a0a974f7bcb3b1051c5a4c9ce400f Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 07:13:41 +0800 Subject: [PATCH 10/11] fix(pchronicle-web): use icon JSON tree --- pchronicle-web/assets/components.css | 105 +++++++++++++++++---------- pchronicle-web/src/json_value.rs | 65 ++++++++--------- pchronicle-web/src/workspace.rs | 4 +- 3 files changed, 98 insertions(+), 76 deletions(-) diff --git a/pchronicle-web/assets/components.css b/pchronicle-web/assets/components.css index 3f4f1ceb..82c5f6aa 100644 --- a/pchronicle-web/assets/components.css +++ b/pchronicle-web/assets/components.css @@ -482,87 +482,118 @@ color: #344054; } +.pc2-json-root { + min-width: 0; + font: 12px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; +} + .pc2-json-tree { min-width: 0; display: flex; flex-direction: column; gap: 0; - padding-left: 14px; - font: 12px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; -} - -.pc2-json-object::before { - content: "{"; -} - -.pc2-json-object::after { - content: "}"; -} - -.pc2-json-array::before { - content: "["; -} - -.pc2-json-array::after { - content: "]"; } .pc2-json-node { min-width: 0; - margin-left: 4px; - border-left: 1px dotted #cbd5e1; - padding-left: 12px; } .pc2-json-leaf { display: flex; align-items: baseline; - gap: 8px; + gap: 7px; min-width: 0; - padding: 1px 4px; + min-height: 22px; + padding: 1px 3px; } +.pc2-json-root > summary, .pc2-json-node > summary { display: flex; align-items: center; - gap: 8px; + gap: 7px; min-height: 24px; - padding: 1px 4px; + padding: 1px 3px; cursor: pointer; list-style: none; } +.pc2-json-root > summary::before, .pc2-json-node > summary::before { - width: 11px; - color: #667085; + box-sizing: border-box; + display: inline-flex; + width: 14px; + height: 14px; + flex: 0 0 14px; + align-items: center; + justify-content: center; + border: 1px solid #94a3b8; + border-radius: 2px; + background: linear-gradient(#fff, #e2e8f0); + color: #475569; content: "+"; - font-weight: 700; - text-align: center; + font: 700 11px/12px sans-serif; } +.pc2-json-root[open] > summary::before, .pc2-json-node[open] > summary::before { content: "−"; } +.pc2-json-root > summary:hover, .pc2-json-node > summary:hover { background: #f1f5f9; } +.pc2-json-root > summary::-webkit-details-marker, .pc2-json-node > summary::-webkit-details-marker { display: none; } +.pc2-json-root > .pc2-json-tree, +.pc2-json-node > .pc2-json-tree { + margin-left: 7px; + padding-left: 20px; + border-left: 1px dotted #cbd5e1; +} + +.pc2-json-kind { + color: #4338ca; + font-size: 15px; + font-weight: 800; +} + +.pc2-json-leaf-icon { + box-sizing: border-box; + width: 9px; + height: 9px; + flex: 0 0 9px; + border: 1px solid #cbd5e1; + box-shadow: inset 0 0 0 1px #ffffff80; +} + +.pc2-json-leaf-icon.string { + background: #334ea2; +} + +.pc2-json-leaf-icon.number { + background: #2f8f46; +} + +.pc2-json-leaf-icon.boolean { + background: #d4b106; +} + +.pc2-json-leaf-icon.null { + background: #c9362b; +} + .pc2-json-key { color: #101828; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 11px; } -.pc2-json-size { - color: #667085; - font-size: 9px; -} - .pc2-json-punctuation { color: #667085; } @@ -623,13 +654,7 @@ word-break: break-word; } -.pc2-json-type { - color: #98a2b3; - font: 9px ui-monospace, SFMono-Regular, Menlo, monospace; -} - .pc2-json-node > .pc2-json-table, -.pc2-json-node > .pc2-json-tree, .pc2-json-node > .pc2-json-scroll, .pc2-json-node > .pc2-json-scalar { margin: 6px 0 8px; diff --git a/pchronicle-web/src/json_value.rs b/pchronicle-web/src/json_value.rs index 7ddfe5f5..37deb22c 100644 --- a/pchronicle-web/src/json_value.rs +++ b/pchronicle-web/src/json_value.rs @@ -76,17 +76,6 @@ pub fn record_columns(rows: &[Value]) -> Vec { columns } -pub fn json_summary(value: &Value) -> String { - match peel_json(value) { - Value::Object(object) => format!("{{{} keys}}", object.len()), - Value::Array(items) => format!("[{} items]", items.len()), - Value::String(_) => "string".into(), - Value::Number(_) => "number".into(), - Value::Bool(_) => "boolean".into(), - Value::Null => "null".into(), - } -} - fn scalar_text(value: &Value) -> String { match value { Value::Null => "null".into(), @@ -125,6 +114,14 @@ fn json_type(value: &Value) -> &'static str { } } +fn json_kind_icon(value: &Value) -> &'static str { + match value { + Value::Array(_) => "[]", + Value::Object(_) => "{}", + _ => "", + } +} + #[component] pub fn JsonValue(value: Value, #[props(default = false)] default_open: bool) -> Element { let peeled = peel_json(&value); @@ -139,6 +136,21 @@ pub fn JsonValue(value: Value, #[props(default = false)] default_open: bool) -> } } +#[component] +pub fn JsonViewer(value: Value) -> Element { + let peeled = peel_json(&value); + let kind = json_type(&peeled); + rsx! { + details { class: "pc2-json-root {kind}", open: true, + summary { + span { class: "pc2-json-kind", "{json_kind_icon(&peeled)}" } + strong { "JSON" } + } + JsonTree { value: peeled, default_open: false } + } + } +} + #[component] fn JsonScalar(value: Value) -> Element { let literal = json_literal(&value); @@ -214,20 +226,16 @@ fn JsonRecordTable(value: Value, default_open: bool) -> Element { #[component] fn JsonTree(value: Value, default_open: bool) -> Element { match value { - Value::Array(items) if items.is_empty() => rsx! { - details { class: "pc2-json-node", open: default_open, - summary { span { class: "pc2-json-size", "[0 items]" } } - } - }, + Value::Array(items) if items.is_empty() => rsx! { div { class: "pc2-json-tree" } }, Value::Object(map) => rsx! { - div { class: "pc2-json-tree pc2-json-object", + div { class: "pc2-json-tree", for (key, child) in map { JsonTreeNode { key: "{key}", label: key, value: child, default_open } } } }, Value::Array(items) => rsx! { - div { class: "pc2-json-tree pc2-json-array", + div { class: "pc2-json-tree", for (index, child) in items.into_iter().enumerate() { JsonTreeNode { key: "{index}", label: String::new(), value: child, default_open: false } } @@ -243,22 +251,23 @@ fn JsonTree(value: Value, default_open: bool) -> Element { #[component] fn JsonTreeNode(label: String, value: Value, default_open: bool) -> Element { let peeled = peel_json(&value); + let kind = json_type(&peeled); if is_scalar(&peeled) { return rsx! { div { class: "pc2-json-leaf", + span { class: "pc2-json-leaf-icon {kind}" } if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } JsonScalar { value: peeled } } }; } - let summary = json_summary(&value); rsx! { - details { class: "pc2-json-node", open: default_open, + details { class: "pc2-json-node {kind}", open: default_open, summary { + span { class: "pc2-json-kind", "{json_kind_icon(&peeled)}" } if !label.is_empty() { span { class: "pc2-json-key", "{label}" span { class: "pc2-json-punctuation", ":" } } } - span { class: "pc2-json-size", "{summary}" } } - JsonValue { value, default_open: false } + JsonTree { value: peeled, default_open: false } } } } @@ -326,18 +335,6 @@ mod tests { assert_eq!(record_columns(&rows), vec!["a", "b", "c"]); } - #[test] - fn json_summary_peels_and_names_types() { - assert_eq!(json_summary(&json!({"a": 1, "b": 2})), "{2 keys}"); - assert_eq!(json_summary(&json!([1, 2, 3])), "[3 items]"); - assert_eq!(json_summary(&json!([])), "[0 items]"); - assert_eq!(json_summary(&json!("{}")), "{0 keys}"); - assert_eq!(json_summary(&json!("hello")), "string"); - assert_eq!(json_summary(&json!(true)), "boolean"); - assert_eq!(json_summary(&json!(1)), "number"); - assert_eq!(json_summary(&json!(null)), "null"); - } - #[test] fn long_json_values_get_single_line_previews() { let value = "x".repeat(JSON_VALUE_PREVIEW_LIMIT + 1); diff --git a/pchronicle-web/src/workspace.rs b/pchronicle-web/src/workspace.rs index 0c2223ea..bee89edf 100644 --- a/pchronicle-web/src/workspace.rs +++ b/pchronicle-web/src/workspace.rs @@ -18,7 +18,7 @@ use crate::copilot_sessions::{ page_after_history_switch, persist_indexed_thread, relative_time, restore_for_run, save_index, session_storage_key, title_from_thread, }; -use crate::json_value::JsonValue; +use crate::json_value::JsonViewer; use crate::llm; use crate::llm_settings::LlmSettings; #[cfg(test)] @@ -1488,7 +1488,7 @@ fn RunDetailWorkspace( if loading && compact_record.is_none() { div { class: "pc2-inline-loading", span { class: "spinner" } "Loading record…" } } else if let Some(detail) = compact_record { - JsonValue { value: detail.record, default_open: true } + JsonViewer { value: detail.record } } else { div { class: "pc2-empty", strong { "Record unavailable" } } } From 72f15123e4cb7b2480a4f45facff673818c925a2 Mon Sep 17 00:00:00 2001 From: Reiase Date: Mon, 7 Sep 2026 07:29:03 +0800 Subject: [PATCH 11/11] feat(pchronicle-web): replace catalog treemap with folders --- .../src/server/explorer.rs | 62 +++++++ .../src/server/mod.rs | 58 ++++++- pchronicle-web/assets/catalog.css | 27 ++- pchronicle-web/src/catalog.rs | 155 +++--------------- pchronicle-web/src/model.rs | 4 + 5 files changed, 161 insertions(+), 145 deletions(-) diff --git a/crates/persisting-pchronicle-cli/src/server/explorer.rs b/crates/persisting-pchronicle-cli/src/server/explorer.rs index 3e8b14de..423c2407 100644 --- a/crates/persisting-pchronicle-cli/src/server/explorer.rs +++ b/crates/persisting-pchronicle-cli/src/server/explorer.rs @@ -52,9 +52,12 @@ pub(crate) struct CatalogTree { pub(crate) struct CatalogTreeChild { pub(crate) name: String, pub(crate) kind: String, + pub(crate) data_type: String, pub(crate) path: String, pub(crate) run_count: usize, pub(crate) failed_count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) total_tokens: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) entries: Vec, } @@ -105,6 +108,27 @@ struct ChildAcc { run_count: usize, failed_count: usize, has_deeper: bool, + data_types: BTreeSet, +} + +fn data_type(format: Option<&str>) -> &'static str { + match format { + Some("compact-jsonl/v1") => "compact-jsonl", + Some("storyline-lance") | None => "storyline", + Some(_) => "other", + } +} + +fn combined_data_type(data_types: &BTreeSet) -> String { + match data_types.len() { + 0 => "unknown".into(), + 1 => data_types + .iter() + .next() + .cloned() + .unwrap_or_else(|| "unknown".into()), + _ => "mixed".into(), + } } fn dataset_children(runs: &[&RunSummary]) -> Vec { @@ -114,8 +138,12 @@ fn dataset_children(runs: &[&RunSummary]) -> Vec { run_count: 0, failed_count: 0, has_deeper: false, + data_types: BTreeSet::new(), }); entry.run_count += 1; + entry + .data_types + .insert(data_type(run.format.as_deref()).into()); if is_failed_status(&run.status) { entry.failed_count += 1; } @@ -125,9 +153,11 @@ fn dataset_children(runs: &[&RunSummary]) -> Vec { .map(|(name, acc)| CatalogTreeChild { name: name.clone(), kind: "dataset".into(), + data_type: combined_data_type(&acc.data_types), path: name, run_count: acc.run_count, failed_count: acc.failed_count, + total_tokens: None, entries: Vec::new(), }) .collect() @@ -161,8 +191,12 @@ fn file_children(runs: &[&RunSummary], prefix: &str) -> Vec { run_count: 0, failed_count: 0, has_deeper: false, + data_types: BTreeSet::new(), }); entry.run_count += 1; + entry + .data_types + .insert(data_type(run.format.as_deref()).into()); if is_failed_status(&run.status) { entry.failed_count += 1; } @@ -184,6 +218,8 @@ fn file_children(runs: &[&RunSummary], prefix: &str) -> Vec { name, run_count: acc.run_count, failed_count: acc.failed_count, + data_type: combined_data_type(&acc.data_types), + total_tokens: None, entries: Vec::new(), }) .collect() @@ -208,14 +244,40 @@ fn fold_tree_children( children.push(CatalogTreeChild { name: "other".into(), kind: "other".into(), + data_type: "mixed".into(), path: prefix.to_string(), run_count: rest.iter().map(|child| child.run_count).sum(), failed_count: rest.iter().map(|child| child.failed_count).sum(), + total_tokens: None, entries: rest, }); children } +pub(crate) fn apply_total_tokens( + children: &mut [CatalogTreeChild], + file_tokens: &BTreeMap, +) { + for child in children { + apply_total_tokens(&mut child.entries, file_tokens); + child.total_tokens = if child.entries.is_empty() { + file_tokens + .iter() + .filter(|(file, _)| { + *file == &child.path || file.starts_with(&format!("{}/", child.path)) + }) + .map(|(_, tokens)| *tokens) + .reduce(u64::saturating_add) + } else { + child + .entries + .iter() + .filter_map(|entry| entry.total_tokens) + .reduce(u64::saturating_add) + }; + } +} + #[derive(Clone, Debug, Serialize)] pub(crate) struct ExplorerPage { pub(crate) snapshot: PageSnapshot, diff --git a/crates/persisting-pchronicle-cli/src/server/mod.rs b/crates/persisting-pchronicle-cli/src/server/mod.rs index 865f135d..723e8200 100644 --- a/crates/persisting-pchronicle-cli/src/server/mod.rs +++ b/crates/persisting-pchronicle-cli/src/server/mod.rs @@ -843,6 +843,8 @@ async fn explorer_tree( let (duration_ms, total_tokens) = tree_prefix_metrics(&runtime, &name, &tree.prefix).await; tree.duration_ms = duration_ms; tree.total_tokens = total_tokens; + let file_tokens = tree_file_tokens(&runtime, &name, &tree.prefix).await; + explorer::apply_total_tokens(&mut tree.children, &file_tokens); } Ok(Json(tree)) } @@ -870,7 +872,11 @@ async fn tree_prefix_metrics( format!(" WHERE _file_ = '{escaped}' OR _file_ LIKE '{escaped}/%'") }; let sql = format!( - "SELECT MIN(timestamp) AS start_ts, MAX(timestamp) AS end_ts FROM {ident}.steps{file_clause}" + "SELECT MIN(timestamp) AS start_ts, MAX(timestamp) AS end_ts, SUM(COALESCE(\ + json_get_int(metrics, 'total_tokens'),\ + json_get_int(metrics, 'prompt_tokens') + json_get_int(metrics, 'completion_tokens'),\ + json_get_int(metrics, 'prompt_tokens_len') + json_get_int(metrics, 'completion_tokens_len')\ + )) AS total_tokens FROM {ident}.steps{file_clause}" ); let mut buffer = Vec::new(); let write = tokio::time::timeout( @@ -894,6 +900,56 @@ async fn tree_prefix_metrics( ) } +async fn tree_file_tokens( + runtime: &CatalogRuntime, + dataset: &str, + prefix: &str, +) -> BTreeMap { + let Some(ident) = sql_ident(dataset) else { + return BTreeMap::new(); + }; + let file_clause = if prefix.is_empty() { + String::new() + } else { + let escaped = prefix.replace('\'', "''"); + format!(" WHERE _file_ = '{escaped}' OR _file_ LIKE '{escaped}/%'") + }; + let sql = format!( + "SELECT _file_, SUM(COALESCE(\ + json_get_int(metrics, 'total_tokens'),\ + json_get_int(metrics, 'prompt_tokens') + json_get_int(metrics, 'completion_tokens'),\ + json_get_int(metrics, 'prompt_tokens_len') + json_get_int(metrics, 'completion_tokens_len')\ + )) AS total_tokens FROM {ident}.steps{file_clause} GROUP BY _file_" + ); + let mut buffer = Vec::new(); + let write = tokio::time::timeout( + Duration::from_secs(3), + runtime + .engine + .write_query_jsonl_with_max_rows(&sql, &mut buffer, None), + ) + .await; + let Ok(Ok(())) = write else { + return BTreeMap::new(); + }; + buffer + .split(|byte| *byte == b'\n') + .filter_map(|line| { + let row: Value = serde_json::from_slice(line).ok()?; + let file = row.get("_file_")?.as_str()?.to_owned(); + let tokens = row + .get("total_tokens") + .and_then(Value::as_u64) + .or_else(|| { + row.get("total_tokens") + .and_then(Value::as_i64) + .map(|value| value.max(0) as u64) + })?; + Some((file, tokens)) + }) + .collect() +} + fn timestamp_span_ms(start: Option<&Value>, end: Option<&Value>) -> Option { let start = json_timestamp_ms(start?)?; let end = json_timestamp_ms(end?)?; diff --git a/pchronicle-web/assets/catalog.css b/pchronicle-web/assets/catalog.css index e6d8a623..f19955b5 100644 --- a/pchronicle-web/assets/catalog.css +++ b/pchronicle-web/assets/catalog.css @@ -11,20 +11,19 @@ .pc-catalog-stats strong{color:#101828;font:600 18px ui-monospace,SFMono-Regular,Menlo,monospace} .pc-catalog-errors{margin:0 0 10px;color:#b42318;font-size:11px} .pc-catalog-mosaic{min-height:0;flex:1;display:flex;flex-direction:column} -.pc-catalog-tree{position:relative;min-height:0;flex:1;border:1px solid #dfe3e8;border-radius:12px;background:#eef2f6;overflow:hidden} -.pc-catalog-tree.compact{flex:0 0 auto;display:flex;flex-wrap:wrap;gap:10px;padding:12px;overflow:visible} -.pc-catalog-tree.compact .pc-catalog-tile{position:static;width:260px;height:120px} -.pc-catalog-tile{position:absolute;display:flex;flex-direction:column;align-items:flex-start;justify-content:space-between;padding:10px;border:1px solid #ffffffaa;border-radius:8px;color:#102033;text-align:left;cursor:pointer;overflow:hidden} -.pc-catalog-tile strong{max-width:100%;overflow:hidden;font-size:13px;text-overflow:ellipsis;white-space:nowrap} -.pc-catalog-tile small{color:#1f2937cc;font:11px ui-monospace,SFMono-Regular,Menlo,monospace} -.pc-catalog-tile.tone-0{background:#93c5fd} -.pc-catalog-tile.tone-1{background:#6ea8ff} -.pc-catalog-tile.tone-2{background:#67e8f9} -.pc-catalog-tile.tone-3{background:#86efac} -.pc-catalog-tile.tone-4{background:#fde68a} -.pc-catalog-tile.tone-5{background:#e2e8f0} -.pc-catalog-tile.kind-other{background:#cbd5e1} -.pc-catalog-tile:hover,.pc-catalog-tile:focus-visible{outline:0;box-shadow:inset 0 0 0 2px #1d4ed8} +.pc-catalog-folders{display:grid;grid-template-columns:repeat(auto-fill,minmax(230px,1fr));gap:12px;min-height:220px;padding:14px;border:1px solid #dfe3e8;border-radius:12px;background:#f8fafc;overflow:auto} +.pc-catalog-folder{display:flex;min-height:132px;flex-direction:column;align-items:flex-start;justify-content:space-between;padding:14px;border:1px solid #d0d5dd;border-radius:10px;background:#fff;color:#102033;text-align:left;cursor:pointer;box-shadow:0 1px 2px #1018280d} +.pc-catalog-folder-title{display:flex;align-items:center;gap:9px;width:100%;min-width:0} +.pc-catalog-folder-title strong{overflow:hidden;font-size:14px;text-overflow:ellipsis;white-space:nowrap} +.pc-catalog-folder-icon{display:grid;width:28px;height:24px;flex:none;place-items:center;border-radius:6px;background:#dbeafe;color:#2563eb;font-size:16px} +.pc-catalog-folder-type{padding:3px 7px;border-radius:999px;background:#eef2ff;color:#4338ca;font:700 9px ui-monospace,SFMono-Regular,Menlo,monospace} +.pc-catalog-folder-meta{display:flex;justify-content:space-between;gap:12px;width:100%;color:#667085;font:11px ui-monospace,SFMono-Regular,Menlo,monospace} +.pc-catalog-folder.type-compact-jsonl{border-top:3px solid #06b6d4} +.pc-catalog-folder.type-storyline{border-top:3px solid #2563eb} +.pc-catalog-folder.type-mixed{border-top:3px solid #8b5cf6} +.pc-catalog-folder.type-other{border-top:3px solid #f59e0b} +.pc-catalog-folder.kind-other{background:#f1f5f9} +.pc-catalog-folder:hover,.pc-catalog-folder:focus-visible{outline:0;box-shadow:0 0 0 2px #1d4ed8,0 4px 12px #10182814} .pc-catalog-empty{min-height:220px;flex:1;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:6px;border:1px dashed #d0d5dd;border-radius:12px;color:#98a2b3;text-align:center} .pc-catalog-empty strong{color:#475467;font-size:13px} .pc-catalog-empty span{font-size:11px} diff --git a/pchronicle-web/src/catalog.rs b/pchronicle-web/src/catalog.rs index 93188844..d3167b35 100644 --- a/pchronicle-web/src/catalog.rs +++ b/pchronicle-web/src/catalog.rs @@ -2,66 +2,6 @@ use dioxus::prelude::*; use crate::model::{CatalogTree, CatalogTreeChild}; -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct TileBox { - pub x: f64, - pub y: f64, - pub w: f64, - pub h: f64, -} - -pub fn layout_treemap(sizes: &[f64], width: f64, height: f64) -> Vec { - if sizes.is_empty() || width <= 0.0 || height <= 0.0 { - return Vec::new(); - } - let total: f64 = sizes.iter().copied().sum(); - if total <= 0.0 { - return sizes - .iter() - .map(|_| TileBox { - x: 0.0, - y: 0.0, - w: 0.0, - h: 0.0, - }) - .collect(); - } - split(sizes, 0.0, 0.0, width, height) -} - -fn split(areas: &[f64], x: f64, y: f64, w: f64, h: f64) -> Vec { - match areas { - [] => Vec::new(), - [_] => vec![TileBox { x, y, w, h }], - _ => { - let total: f64 = areas.iter().sum(); - let mut acc = 0.0; - let mut cut = 1; - for (index, area) in areas.iter().enumerate() { - acc += area; - cut = index + 1; - if acc >= total / 2.0 { - break; - } - } - cut = cut.clamp(1, areas.len() - 1); - let left_sum: f64 = areas[..cut].iter().sum(); - let frac = left_sum / total; - if w >= h { - let left = w * frac; - let mut tiles = split(&areas[..cut], x, y, left, h); - tiles.extend(split(&areas[cut..], x + left, y, w - left, h)); - tiles - } else { - let top = h * frac; - let mut tiles = split(&areas[..cut], x, y, w, top); - tiles.extend(split(&areas[cut..], x, y + top, w, h - top)); - tiles - } - } - } -} - #[component] pub fn CatalogExplorer( tree: Option, @@ -111,7 +51,7 @@ pub fn CatalogExplorer( span { "This path contains one source file. Open it in Runs to inspect its runs." } } } else { - CatalogMosaic { + CatalogFolders { tree: tree.clone().unwrap(), on_open, on_runs, @@ -208,46 +148,23 @@ fn CatalogStats(tree: Option) -> Element { } #[component] -fn CatalogMosaic( +fn CatalogFolders( tree: CatalogTree, on_open: EventHandler<(String, String)>, on_runs: EventHandler<(String, String)>, on_other: EventHandler, ) -> Element { - let sizes = tree - .children - .iter() - .map(|child| child.run_count.max(1) as f64) - .collect::>(); - let boxes = layout_treemap(&sizes, 100.0, 100.0); let dataset = tree.dataset.clone().unwrap_or_default(); - // A treemap with one or two children stretches a single tile across the - // whole viewport. Render those as fixed-size cards instead. - let compact = tree.children.len() <= 2; - let tree_class = if compact { - "pc-catalog-tree compact" - } else { - "pc-catalog-tree" - }; rsx! { - div { class: "{tree_class}", - for (index, child) in tree.children.iter().cloned().enumerate() { - { - let tile = boxes.get(index).copied().unwrap_or(TileBox { x: 0.0, y: 0.0, w: 0.0, h: 0.0 }); - let tone = index % 6; - rsx! { - CatalogTile { - key: "{child.kind}:{child.path}", - child, - dataset: dataset.clone(), - tile, - tone, - compact, - on_open, - on_runs, - on_other, - } - } + div { class: "pc-catalog-folders", + for child in tree.children.iter().cloned() { + CatalogFolder { + key: "{child.kind}:{child.path}", + child, + dataset: dataset.clone(), + on_open, + on_runs, + on_other, } } } @@ -255,32 +172,22 @@ fn CatalogMosaic( } #[component] -fn CatalogTile( +fn CatalogFolder( child: CatalogTreeChild, dataset: String, - tile: TileBox, - tone: usize, - compact: bool, on_open: EventHandler<(String, String)>, on_runs: EventHandler<(String, String)>, on_other: EventHandler, ) -> Element { - let style = if compact { - String::new() - } else { - format!( - "left:{:.3}%;top:{:.3}%;width:{:.3}%;height:{:.3}%;", - tile.x, tile.y, tile.w, tile.h - ) - }; let kind = child.kind.clone(); let path = child.path.clone(); let name = child.name.clone(); + let data_type = child.data_type.clone(); + let tokens = format_tokens(child.total_tokens); rsx! { button { - class: "pc-catalog-tile tone-{tone} kind-{kind}", - style, - title: "{name} · {child.run_count} runs", + class: "pc-catalog-folder type-{data_type} kind-{kind}", + title: "{name} · {child.run_count} trajectories · {tokens} tokens", onclick: move |event| { match kind.as_str() { "other" => on_other.call(event), @@ -289,8 +196,15 @@ fn CatalogTile( _ => on_open.call((dataset.clone(), path.clone())), } }, - strong { "{child.name}" } - small { "{child.run_count}" } + div { class: "pc-catalog-folder-title", + span { class: "pc-catalog-folder-icon", if child.kind == "file" { "▤" } else { "▰" } } + strong { "{child.name}" } + } + span { class: "pc-catalog-folder-type", "{data_type}" } + div { class: "pc-catalog-folder-meta", + span { "{child.run_count} trajectories" } + span { "{tokens} tokens" } + } } } } @@ -366,22 +280,3 @@ fn format_tokens(tokens: Option) -> String { tokens.to_string() } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn treemap_covers_the_canvas_and_keeps_area_proportional() { - let boxes = layout_treemap(&[2.0, 1.0, 1.0], 100.0, 100.0); - assert_eq!(boxes.len(), 3); - let area: f64 = boxes.iter().map(|tile| tile.w * tile.h).sum(); - assert!((area - 10_000.0).abs() < 0.01); - assert!((boxes[0].w * boxes[0].h - 5_000.0).abs() < 0.01); - } - - #[test] - fn empty_sizes_yield_no_tiles() { - assert!(layout_treemap(&[], 100.0, 100.0).is_empty()); - } -} diff --git a/pchronicle-web/src/model.rs b/pchronicle-web/src/model.rs index f1c6ae03..a6a9a4dd 100644 --- a/pchronicle-web/src/model.rs +++ b/pchronicle-web/src/model.rs @@ -137,12 +137,16 @@ pub struct CatalogTreeChild { pub name: String, pub kind: String, #[serde(default)] + pub data_type: String, + #[serde(default)] pub path: String, #[serde(default)] pub run_count: usize, #[serde(default)] pub failed_count: usize, #[serde(default)] + pub total_tokens: Option, + #[serde(default)] pub entries: Vec, }