From a392f9a539219762fd74c196851578a8fc5465c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 5 Sep 2026 10:33:51 +0000 Subject: [PATCH 1/8] refactor(lib): persist only critical and creation commit envelopes Ordinary content commits are applied and then discarded. Genesis, rights, parent, and destroy stay, including unflagged creations. Drop previous-commit validation, stop indexing Loro binaries as KV keys, and stop creating the /commits collection. Co-authored-by: joepmeindertsma --- flutter/rust/src/api/simple.rs | 1 - lib/defaults/default_store.json | 9 +- lib/src/collections.rs | 6 -- lib/src/commit.rs | 124 +++++------------------ lib/src/db.rs | 19 ++-- lib/src/db/test.rs | 171 +++++++++++++++++++------------- lib/src/hierarchy.rs | 4 - lib/src/loro.rs | 2 - lib/src/parse.rs | 2 - lib/src/populate.rs | 5 +- lib/src/resources.rs | 6 +- lib/src/storelike.rs | 6 +- lib/src/sync/engine.rs | 13 +-- lib/src/sync/peer.rs | 1 - lib/src/sync/tests.rs | 4 - lib/src/test_utils.rs | 4 + lib/src/values.rs | 20 +++- server/src/plugins/wasm.rs | 1 - 18 files changed, 177 insertions(+), 221 deletions(-) diff --git a/flutter/rust/src/api/simple.rs b/flutter/rust/src/api/simple.rs index 838523d7d..27264797e 100644 --- a/flutter/rust/src/api/simple.rs +++ b/flutter/rust/src/api/simple.rs @@ -651,7 +651,6 @@ async fn destroy_resource_and_sync(subject: String) -> Result<(), String> { let opts = atomic_lib::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..atomic_lib::commit::CommitOpts::no_validations_no_index() diff --git a/lib/defaults/default_store.json b/lib/defaults/default_store.json index 333ce7303..189bf688a 100644 --- a/lib/defaults/default_store.json +++ b/lib/defaults/default_store.json @@ -740,7 +740,7 @@ "@id": "https://atomicdata.dev/properties/lastCommit", "https://atomicdata.dev/properties/classtype": "https://atomicdata.dev/classes/Commit", "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/atomicURL", - "https://atomicdata.dev/properties/description": "The last Commit that was applied to this Resource. This is used when checking whether two systems have the same version of a resource.", + "https://atomicdata.dev/properties/description": "The id of the last Commit applied to this Resource (`did:ad:commit:{signature}`). A stamp for echo-dedup and genesis detection. Not a refetchable resource — ordinary content commits are discarded after apply.", "https://atomicdata.dev/properties/isA": [ "https://atomicdata.dev/classes/Property" ], @@ -782,7 +782,7 @@ "@id": "https://atomicdata.dev/properties/previousCommit", "https://atomicdata.dev/properties/classtype": "https://atomicdata.dev/classes/Commit", "https://atomicdata.dev/properties/datatype": "https://atomicdata.dev/datatypes/atomicURL", - "https://atomicdata.dev/properties/description": "The previous Commit that was applied to the target resource (the subject) of this Commit. You should be able to follow these from Commit to Commit to establish an audit trail.", + "https://atomicdata.dev/properties/description": "Optional audit pointer at an earlier envelope. Not a causal gate — concurrent edits merge via Loro. Clients may omit it; servers do not require it.", "https://atomicdata.dev/properties/isA": [ "https://atomicdata.dev/classes/Property" ], @@ -1014,7 +1014,7 @@ }, { "@id": "https://atomicdata.dev/classes/Commit", - "https://atomicdata.dev/properties/description": "A Commit is a Resource that describes how a Resource must be updated.\nIt can be used for auditing, versioning and feeds.\nIt is cryptographically signed by an [Agent](https://atomicdata.dev/classes/Agent).\n\nThe **required fields** are:\n\n- `subject` - The thing being changed. A Resource Subject URL (HTTP identifier) that the Commit is changing about. A Commit Subject must not contain query parameters, as these are reserved for dynamic resources.\n- `signer` - Who's making the change. The Atomic URL of the Author's profile - which in turn must contain a `publicKey`.\n- `signature` - Cryptographic proof of the change. A hash of the JSON-AD serialized Commit (without the `signature` field), signed by the Agent's `private-key`. This proves that the author is indeed the one who created this exact commit. The signature of the Commit is also used as the identifier of the commit.\n- `created-at` - When the change was made. A UNIX timestamp number of when the commit was created.\n\nThe **optional method fields** describe how the data must be changed:\n\n- `destroy` - If true, the existing Resource will be removed.\n- `remove` - an array of Properties that need to be removed (including their values).\n- `set` - a Nested Resource which contains all the new or edited fields.\n\nThese commands are executed in the order above.\nThis means that you can set `destroy` to `true` and include `set`, which empties the existing resource and sets new values.\n\n### Posting commits using HTTP\n\nSince Commits contains cryptographic proof of authorship, they can be accepted at a public endpoint.\nThere is no need for authentication.\n\nA commit should be sent (using an HTTPS POST request) to a `/commmit` endpoint of an Atomic Server.\nThe server then checks the signature and the author rights, and responds with a `2xx` status code if it succeeded, or an `5xx` error if something went wrong.\nThe error will be a JSON object.\n\n### Serialization with JSON-AD\n\nLet's look at an example Commit:\n\n```json\n{\n \"@id\": \"https://atomicdata.dev/commits/3n+U/3OvymF86Ha6S9MQZtRVIQAAL0rv9ZQpjViht4emjnqKxj4wByiO9RhfL+qwoxTg0FMwKQsNg6d0QU7pAw==\",\n \"https://atomicdata.dev/properties/createdAt\": 1611489929370,\n \"https://atomicdata.dev/properties/isA\": [\n \"https://atomicdata.dev/classes/Commit\"\n ],\n \"https://atomicdata.dev/properties/set\": {\n \"https://atomicdata.dev/properties/shortname\": \"1611489928\"\n },\n \"https://atomicdata.dev/properties/signature\": \"3n+U/3OvymF86Ha6S9MQZtRVIQAAL0rv9ZQpjViht4emjnqKxj4wByiO9RhfL+qwoxTg0FMwKQsNg6d0QU7pAw==\",\n \"https://atomicdata.dev/properties/signer\": \"https://surfy.ddns.net/agents/9YCs7htDdF4yBAiA4HuHgjsafg+xZIrtZNELz4msCmc=\",\n \"https://atomicdata.dev/properties/subject\": \"https://atomicdata.dev/test\"\n}\n```\n\nThis Commit can be sent to any Atomic Server.\nThis server, in turn, should verify the signature and the author's rights before the server applies the Commit.\n\n### Calculating the signature\n\nThe signature is a base64 encoded Ed25519 signature of the deterministically serialized Commit.\nCalculating the signature is a delicate process that should be followed to the letter - even a single character in the wrong place will result in an incorrect signature, which makes the Commit invalid.\n\nThe first step is **serializing the commit deterministically**.\nThis means that the process will always end in the exact same string.\n\n- Serialize the Commit as JSON-AD.\n- Do not serialize the signature field.\n- Do not include empty objects or arrays.\n- If `destroy` is false, do not include it.\n- All keys are sorted alphabetically - both in the root object, as in any nested objects.\n- The JSON-AD is minified: no newlines, no spaces.\n\nThis will result in a string.\nThe next step is to sign this string using the Ed25519 private key from the Author.\nThis signature is a byte array, which should be encoded in base64 for serialization.\nMake sure that the Author's URL resolves to a Resource that contains the linked public key.\n\nCongratulations, you've just created a valid Commit!\n\nHere are currently working implementations of this process, including serialization and signing (links are permalinks).\n\n- [in Rust (atomic-lib)](https://github.com/atomicdata-dev/atomic-server/blob/ceb88c1ae58811f2a9e6bacb7eaa39a2a7aa1513/lib/src/commit.rs#L81).\n- [in Typescript / Javascript (atomic-data-browser)](https://github.com/atomicdata-dev/atomic-data-browser/blob/fc899bb2cf54bdff593ee6b4debf52e20a85619e/src/atomic-lib/commit.ts#L51).\n\nIf you want validate your implementation, check out the tests for these two projects.\n\n### Applying the Commit\n\nIf you're on the receiving end of a Commit (e.g. if you're writing a server or a client who has to parse Commits), you will _apply_ the Commit to your Store.\nIf you have to _persist_ the Commit, you must perform all of the checks.\nIf you're writing a client, and you trust the source of the Commit, you can probably skip the validation steps.\n\nHere's how you apply a Commit:\n\n1. Check if the Subject URL is valid\n2. Validate the signature. This means serialize the Commit deterministically (see above), check the Agent's publickey (you might need to fetch this one), verify if the signature matches.\n3. Check if the timestamp matches is OK. I think an acceptable window is 10 seconds.\n4. If the Commit is for an existing resource, get it.\n5. Validate the Rights of the one making the Commit.\n6. Check if the `previousCommit` of the Commit matches with the `previousCommit` of the Resource.\n7. Iterate over the `set` fields. Overwrite existing, or add the new Values. Make sure the Datatypes match with the respective Properties.\n8. Iterate over the `remove` fields. Remove existing properties.\n9. If the Resource has one or more classes, check if the required Properties are there.\n10. You might want to perform some custom validations now (e.g. if you accept an Invite, you should make sure that the one creating the Invite has the correct rights to actually make it!)\n11. Store the created Commit as a Resource, and store the modified Resource!\n\n## Limitations\n\n- Commits adjust **only one Resource at a time**, which means that you cannot change multiple in one commit.\n- The one creating the Commit will **need to sign it**, which may make clients that write data more complicated than you'd like. You can also let Servers write Commits, but this makes them less verifiable / decentralized.\n- Commits require signatures, which means **key management**. Doing this securely is no trivial matter.\n- The signatures **require JSON-AD** serialization\n- If your implementation persists all Commits, you might need to **store a lot of data**.\n", + "https://atomicdata.dev/properties/description": "A signed envelope wrapping a Loro CRDT update. Used to authorize writes. Not a queryable event log — current state lives in the resource's Loro document.", "https://atomicdata.dev/properties/isA": [ "https://atomicdata.dev/classes/Class" ], @@ -1022,9 +1022,6 @@ "https://atomicdata.dev/properties/destroy", "https://atomicdata.dev/properties/isGenesis", "https://atomicdata.dev/properties/previousCommit", - "https://atomicdata.dev/properties/remove", - "https://atomicdata.dev/properties/set", - "https://atomicdata.dev/properties/push", "https://atomicdata.dev/properties/loroUpdate" ], "https://atomicdata.dev/properties/requires": [ diff --git a/lib/src/collections.rs b/lib/src/collections.rs index 51b136538..f8e0d423f 100644 --- a/lib/src/collections.rs +++ b/lib/src/collections.rs @@ -623,17 +623,11 @@ pub async fn create_collection_resource_for_class( let mut collection = CollectionBuilder::class_collection(&class.subject, &pluralized, store)?; collection.sort_by = match class_subject { - urls::COMMIT => Some(urls::CREATED_AT.to_string()), urls::CLASS | urls::PROPERTY => Some(urls::SHORTNAME.to_string()), urls::COLLECTION => Some(urls::COLLECTION_VALUE.to_string()), _other => None, }; - collection.sort_desc = match class_subject { - urls::COMMIT => true, - _other => false, - }; - // Agents use DID subjects which are external, so we need to include external resources collection.include_external = match class_subject { urls::AGENT => true, diff --git a/lib/src/commit.rs b/lib/src/commit.rs index afa129c6c..96bc63814 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -32,9 +32,14 @@ impl CommitResponse { /// The authorization relevance of this commit — which authority-defining /// facts it establishes or mutates. See [`crate::hierarchy::AuthImpact`]. pub fn auth_impact(&self) -> crate::hierarchy::AuthImpact { + // A creation is genesis whether or not the client flagged it: Rust + // `save_locally`, agent first-commits and HTTP-subject creations + // arrive with `is_genesis: None`, and the commit that brought a + // resource into being is retained like an explicit genesis. + let created = self.resource_old.is_none() && self.resource_new.is_some(); crate::hierarchy::classify_auth_impact( &self.changed_props, - self.commit.is_genesis == Some(true), + self.commit.is_genesis == Some(true) || created, self.commit.destroy.unwrap_or(false), ) } @@ -70,9 +75,6 @@ pub struct CommitOpts { pub validate_timestamp: bool, /// Checks whether the creator of the Commit has the rights to edit the Resource. pub validate_rights: bool, - /// Checks whether the previous Commit applied to the resource matches the one mentioned in the Commit/ - /// This makes sure that the Commit is not applied twice, or that the one creating it had a faulty state. - pub validate_previous_commit: bool, /// Detects commits whose Loro update's writes silently lost LWW against /// the stored state — i.e. the client's Loro doc wasn't seeded from the /// server's current state, so its ops are concurrent with stored ops and @@ -99,7 +101,6 @@ impl CommitOpts { validate_signature: false, validate_timestamp: false, validate_rights: false, - validate_previous_commit: false, validate_loro_causality: false, update_index: false, validate_for_agent: None, @@ -130,7 +131,7 @@ pub struct Commit { /// Base64 encoded signature of the JSON serialized Commit #[serde(rename = "https://atomicdata.dev/properties/signature")] pub signature: Option, - /// The previously applied commit to this Resource. + /// Optional audit pointer at an earlier envelope. Not a causal gate. #[serde(rename = "https://atomicdata.dev/properties/previousCommit")] pub previous_commit: Option, /// Whether this is the first commit for a Resource. @@ -182,34 +183,6 @@ impl Commit { Ok(()) } - pub fn validate_previous_commit( - &self, - resource_old: &Resource, - subject_url: &str, - ) -> AtomicResult<()> { - let commit = self; - if let Ok(last_commit_val) = resource_old.get(urls::LAST_COMMIT) { - let last_commit = last_commit_val.to_string(); - - if let Some(prev_commit) = commit.previous_commit.clone() { - // TODO: try auto merge - if last_commit != prev_commit { - return Err(format!( - "previousCommit mismatch. Had lastCommit '{}' in Resource {}, but got in Commit '{}'. Perhaps you created the Commit based on an outdated version of the Resource.", - last_commit, subject_url, prev_commit, - ) - .into()); - } - } else { - return Err(format!("Missing `previousCommit`. Resource {} already exists, and it has a `lastCommit` field, so a `previousCommit` field is required in your Commit.", commit.subject).into()); - } - } else { - // If there is no lastCommit in the Resource, we'll accept the Commit. - tracing::warn!("No `lastCommit` in Resource. This can be a bug, or it could be that the resource was never properly updated."); - } - Ok(()) - } - /// Creates a new Commit with a `did:ad` Subject. /// The ID of the Subject is the signature of the Commit. pub async fn create_did( @@ -609,24 +582,10 @@ impl Commit { } } - // `previous_commit` is recorded on every commit for audit / history - // navigation, but it is NOT a validation gate. Concurrency is handled - // by the Loro CRDT itself: each commit's `loro_update` carries the - // op's peer-scoped Lamport clock, and concurrent edits merge - // deterministically — there is no single linear chain to enforce. - // - // The previous behaviour ("commit's `previousCommit` must equal the - // resource's current `lastCommit`") was a Git-style optimistic- - // concurrency check that fought the CRDT semantics: under any real - // concurrent edit (two peers committing without seeing each other), - // one of them would be rejected even though Loro could merge them - // perfectly. It also produced a leaky wire-protocol invariant — the - // client had to round-trip `lastCommit` through every code path or - // its next commit would 500. - // - // The is-genesis distinction below stays — that's about identity - // (subject = signature), not ordering. - let _ = opts.validate_previous_commit; + // `previous_commit` is optional audit metadata. It is NOT a + // validation gate. Concurrency is handled by the Loro CRDT: + // each commit's `loro_update` carries the op's peer-scoped + // Lamport clock, and concurrent edits merge deterministically. // Reject commits that carry no Loro update and aren't a destroy. // Loro is the single source of truth for all user data; a commit @@ -977,7 +936,8 @@ impl Commit { let commit_resource: Resource = commit.into_resource(store).await?; - // Set the `lastCommit` to the newly created Commit + // Stamp `lastCommit` with this envelope's id. The id is a receipt, + // not a refetchable resource — ordinary content commits are not stored. applied .resource_new .set( @@ -1270,7 +1230,7 @@ pub struct CommitBuilder { loro_update: Option>, /// If set to true, deletes the entire resource destroy: bool, - /// The previous Commit that was applied to the target resource (the subject) of this Commit. + /// Optional audit pointer at an earlier envelope. Not a causal gate. previous_commit: Option, /// Whether this is a genesis commit (the first commit for a DID resource). pub is_genesis: bool, @@ -1304,10 +1264,6 @@ impl CommitBuilder { Ok(commit_builder) } - /// Creates the Commit and signs it using a signature. - /// Does not send it - see [atomic_lib::client::post_commit]. - /// Private key is the base64 encoded pkcs8 for the signer. - /// Sets the `previousCommit` using the `lastCommit`. /// Returns true if this builder has any pending change that would /// produce a non-empty commit. Used by callers (`Resource::save`, /// `Resource::save_locally`) to skip a sign+apply round-trip when @@ -1323,15 +1279,17 @@ impl CommitBuilder { || self.destroy } + /// Creates the Commit and signs it using a signature. + /// Does not send it - see [atomic_lib::client::post_commit]. + /// Private key is the base64 encoded pkcs8 for the signer. pub async fn sign( mut self, agent: &crate::agents::Agent, store: &impl Storelike, resource: &Resource, ) -> AtomicResult { - if let Ok(last) = resource.get(urls::LAST_COMMIT) { - self.previous_commit = Some(last.to_string()); - } + // previousCommit is optional audit metadata. Callers that want a + // chain put it on the builder; Loro is the causal authority. // If the resource has a live Loro doc but no snapshot was eagerly // exported to the commit builder, export it now (single export). @@ -1399,7 +1357,7 @@ impl CommitBuilder { self.loro_update = Some(update); } - /// Set the previous Commit URL (for the commit chain on the target resource). + /// Set an optional audit pointer at an earlier envelope. Not a causal gate. pub fn set_previous_commit(&mut self, previous_commit: String) { self.previous_commit = Some(previous_commit); } @@ -1513,7 +1471,6 @@ mod test { validate_schema: true, validate_signature: true, validate_timestamp: true, - validate_previous_commit: true, validate_loro_causality: true, validate_rights: false, validate_for_agent: None, @@ -1541,26 +1498,10 @@ mod test { let value2 = Value::new("someval", &DataType::Slug).unwrap(); commitbuiler.set(property2.into(), value2); let commit = commitbuiler.sign(&agent, &store, &resource).await.unwrap(); - let commit_subject = commit.get_subject().to_string(); let _created_resource = store.apply_commit(commit, &OPTS).await.unwrap(); let resource = store.get_resource(&subject.into()).await.unwrap(); assert!(resource.get(property1).unwrap().to_string() == value1.to_string()); - let found_commit = store - .get_resource(&commit_subject.as_str().into()) - .await - .unwrap(); - println!("Found commit subject: {}", found_commit.get_subject()); - println!("Found commit props: {:?}", found_commit.get_propvals()); - - assert!( - found_commit - .get_shortname("description", &store) - .await - .unwrap() - .to_string() - == value1.to_string() - ); } #[tokio::test] @@ -1738,9 +1679,9 @@ mod test { } { // A did:ad: subject with a subpath is structurally invalid. - // sign() now enforces that did:ad: commits without a previous_commit - // must have is_genesis=true, so we set that here. apply_commit then - // rejects the subpath as "Invalid DID". + // sign() requires is_genesis=true for a new DID resource, so we + // set that here. apply_commit then rejects the subpath as + // "Invalid DID". let subject = "did:ad:cbXxQGm7UBBS5JPvl/NR/p9RJNbSMUjvA7lRYQt9lZvKZrU1FBo6Icl5uctr7i1AMZ/mElWZ3X1dApo5ifzmBg==/subpath"; let mut commitbuilder = crate::commit::CommitBuilder::new(subject.into()); commitbuilder.is_genesis = true; @@ -1791,7 +1732,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, ..CommitOpts::no_validations_no_index() }; @@ -1833,7 +1773,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, validate_loro_causality: true, ..CommitOpts::no_validations_no_index() @@ -1925,7 +1864,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, validate_loro_causality: true, ..CommitOpts::no_validations_no_index() @@ -2008,7 +1946,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, validate_loro_causality: true, ..CommitOpts::no_validations_no_index() @@ -2088,7 +2025,6 @@ mod test { let opts = CommitOpts { validate_signature: false, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, ..CommitOpts::no_validations_no_index() }; @@ -2159,7 +2095,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..CommitOpts::no_validations_no_index() @@ -2202,7 +2137,6 @@ mod test { let opts_no_rights = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, ..CommitOpts::no_validations_no_index() }; @@ -2267,7 +2201,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..CommitOpts::no_validations_no_index() @@ -2427,7 +2360,6 @@ mod test { let opts_with_rights = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: true, validate_rights: true, validate_for_agent: Some(agent.subject.to_string()), update_index: true, @@ -2512,7 +2444,6 @@ mod test { let opts_with_rights = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: true, validate_rights: true, validate_for_agent: Some(agent.subject.to_string()), update_index: true, @@ -2560,7 +2491,6 @@ mod test { let opts_with_rights = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: true, validate_rights: true, validate_for_agent: Some(agent.subject.to_string()), update_index: true, @@ -2605,7 +2535,6 @@ mod test { let opts_with_rights = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: true, validate_rights: true, validate_for_agent: Some(agent.subject.to_string()), update_index: true, @@ -2638,7 +2567,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, ..CommitOpts::no_validations_no_index() }; @@ -2671,7 +2599,6 @@ mod test { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, ..CommitOpts::no_validations_no_index() }; @@ -2755,7 +2682,8 @@ mod test { .unwrap(); let mut builder2 = CommitBuilder::new(subject.into()); builder2.set_loro_update(client_doc.export_snapshot()); - // `sign()` auto-fills previous_commit from the resource's lastCommit. + // previousCommit is optional audit metadata; the TS client still + // sets it, but sign() does not auto-fill a causal chain. let commit2 = builder2.sign(&agent, &store, &after_first).await.unwrap(); store.apply_commit(commit2, &OPTS).await.unwrap(); @@ -2784,7 +2712,6 @@ mod test { validate_schema: true, validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_loro_causality: true, validate_rights: false, validate_for_agent: None, @@ -3007,7 +2934,6 @@ mod owner_mode_tests { CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: true, validate_rights: true, validate_for_agent: Some(agent.subject.to_string()), update_index: true, diff --git a/lib/src/db.rs b/lib/src/db.rs index 83bffeb7f..e45770590 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -876,7 +876,6 @@ impl Db { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() @@ -956,7 +955,6 @@ impl Db { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() @@ -1010,7 +1008,6 @@ impl Db { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() @@ -3138,11 +3135,17 @@ impl Storelike for Db { } } - // Save the Commit to the Store. We can skip the required props checking, but we need to make sure the commit hasn't been applied before. - store.add_resource_tx(&commit_response.commit_resource, &mut transaction)?; - // We still need to index the Commit! - for atom in commit_response.commit_resource.to_atoms() { - store.add_atom_to_index(&atom, &commit_response.commit_resource, &mut transaction)?; + // Commits are signed envelopes, not a queryable class. Keep genesis + // and rights/parent/destroy; drop ordinary content certificates. + if commit_response.auth_impact().is_critical() { + store.add_resource_tx(&commit_response.commit_resource, &mut transaction)?; + for atom in commit_response.commit_resource.to_atoms() { + store.add_atom_to_index( + &atom, + &commit_response.commit_resource, + &mut transaction, + )?; + } } match (&commit_response.resource_old, &commit_response.resource_new) { diff --git a/lib/src/db/test.rs b/lib/src/db/test.rs index 6d623685f..24b15c48d 100644 --- a/lib/src/db/test.rs +++ b/lib/src/db/test.rs @@ -89,8 +89,7 @@ async fn basic() { #[tokio::test] /// Check if a resource is properly removed from the DB after a delete command. -/// Also counts commits. -async fn destroy_resource_and_check_collection_and_commits() { +async fn destroy_resource_and_check_collection() { let store = Db::init_temp("counter").await.unwrap(); crate::test_utils::setup_test_env(&store).await.unwrap(); let for_agent = &ForAgent::Public; @@ -114,20 +113,6 @@ async fn destroy_resource_and_check_collection_and_commits() { "There should be 1 agent in this collection initially (the agent created during init)" ); - // We will count the commits, and check if they've incremented later on. - let commits_url = "internal:/commits".to_string(); - let commits_collection_1 = store - .get_resource_extended(&commits_url.as_str().into(), false, for_agent) - .await - .unwrap(); - let commits_collection_count_1 = commits_collection_1 - .to_single() - .get(crate::urls::COLLECTION_MEMBER_COUNT) - .unwrap() - .to_int() - .unwrap(); - println!("Commits collection count 1: {}", commits_collection_count_1); - // Create a new agent, check if it is added to the new Agents collection as a Member. let mut resource = crate::agents::Agent::new(None) .unwrap() @@ -149,23 +134,6 @@ async fn destroy_resource_and_check_collection_and_commits() { "The new Agent resource did not increase the collection member count from 1 to 2." ); - let commits_collection_2 = store - .get_resource_extended(&commits_url.as_str().into(), false, for_agent) - .await - .unwrap(); - let commits_collection_count_2 = commits_collection_2 - .to_single() - .get(crate::urls::COLLECTION_MEMBER_COUNT) - .unwrap() - .to_int() - .unwrap(); - println!("Commits collection count 2: {}", commits_collection_count_2); - assert_eq!( - commits_collection_count_2, - commits_collection_count_1 + 1, - "The commits collection did not increase after saving the resource." - ); - let clone = _res.resource_new.clone().unwrap(); let resp = _res.resource_new.unwrap().destroy(&store).await.unwrap(); assert!(resp.resource_new.is_none()); @@ -201,23 +169,6 @@ async fn destroy_resource_and_check_collection_and_commits() { agents_collection_count_3, 1, "The collection count did not decrease after destroying the resource." ); - - let commits_collection_3 = store - .get_resource_extended(&commits_url.as_str().into(), false, for_agent) - .await - .unwrap(); - let commits_collection_count_3 = commits_collection_3 - .to_single() - .get(crate::urls::COLLECTION_MEMBER_COUNT) - .unwrap() - .to_int() - .unwrap(); - println!("Commits collection count 3: {}", commits_collection_count_3); - assert_eq!( - commits_collection_count_3, - commits_collection_count_2 + 1, - "The commits collection did not increase after destroying the resource." - ); } /// Regression test for stale parent-index entries leaking into `count`. @@ -392,26 +343,32 @@ async fn get_extended_resource_pagination() { .await .unwrap(); crate::test_utils::setup_test_env(&store).await.unwrap(); - let subject = format!( - "{}/commits?current_page=2&page_size=99999", - "http://localhost" - ); + + // Need enough local members that page 2 exists at page_size=1. This used to + // paginate `/commits` (every write minted a member). The `/commits` + // collection is no longer created; `/agents` has `include_external` and + // DID subjects, so extra agents show up. `/classes` does not: class + // subjects are `https://atomicdata.dev/…` and `include_external` is false. + for _ in 0..5 { + let mut agent = crate::agents::Agent::new(None) + .unwrap() + .to_resource() + .unwrap(); + agent.save_locally(&store).await.unwrap(); + } + let for_agent = &ForAgent::Public; + let too_big = "http://localhost/agents?current_page=2&page_size=99999"; if store - .get_resource_extended(&subject.as_str().into(), false, for_agent) + .get_resource_extended(&too_big.into(), false, for_agent) .await .is_ok() { panic!("Page 2 should not exist, because page size is set to a high value.") } - // let subject = "https://atomicdata.dev/classes?current_page=2&page_size=1"; - let subject_with_page_size = format!("{}&page_size=1", subject); + let paged = "http://localhost/agents?current_page=2&page_size=1"; let resource = store - .get_resource_extended( - &subject_with_page_size.as_str().into(), - false, - &ForAgent::Public, - ) + .get_resource_extended(&paged.into(), false, &ForAgent::Public) .await .unwrap() .to_single(); @@ -421,7 +378,7 @@ async fn get_extended_resource_pagination() { .to_int() .unwrap(); assert_eq!(cur_page, 2); - assert_eq!(resource.get_subject().as_str(), &subject_with_page_size); + assert_eq!(resource.get_subject().as_str(), paged); } /// Generate a bunch of resources, query them. @@ -1384,7 +1341,6 @@ async fn did_loro_only_commit_sled() { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_loro_causality: false, validate_rights: true, validate_schema: true, @@ -1438,7 +1394,6 @@ async fn loro_non_property_container_survives_commit_roundtrip() { let opts = CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_loro_causality: true, validate_rights: true, validate_schema: true, @@ -1866,7 +1821,6 @@ async fn a_cascade_deleted_child_names_its_drive() { validate_signature: true, validate_timestamp: false, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, validate_for_agent: Some(agent.subject.to_string()), update_index: true, @@ -1966,7 +1920,6 @@ async fn find_resource_scoped_to_its_drive() { validate_signature: true, validate_timestamp: false, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, validate_for_agent: Some(agent.subject.to_string()), update_index: true, @@ -2461,3 +2414,87 @@ async fn partial_index_for_an_unwatched_filter_is_rebuilt() { res.count ); } + +#[tokio::test] +#[timeout(120000)] +async fn content_commits_are_not_stored() { + let store = Db::init_temp("content_commits_are_not_stored") + .await + .unwrap(); + + let mut resource = crate::Resource::new("did:ad:placeholder".into()); + resource + .set(urls::NAME.into(), Value::String("first".into()), &store) + .await + .unwrap(); + let genesis = resource.save_as_genesis(&store).await.unwrap(); + let genesis_commit = genesis.commit_resource.get_subject().clone(); + let subject = genesis.resource_new.unwrap().get_subject().clone(); + + assert!( + store.get_resource(&genesis_commit).await.is_ok(), + "genesis commits are always retained" + ); + + let mut resource = store.get_resource(&subject).await.unwrap(); + resource + .set(urls::NAME.into(), Value::String("second".into()), &store) + .await + .unwrap(); + let content = resource.save_locally(&store).await.unwrap(); + let content_commit = content.commit_resource.get_subject().clone(); + assert_eq!( + store + .get_resource(&subject) + .await + .unwrap() + .get(urls::NAME) + .unwrap() + .to_string(), + "second", + "dropping the commit row must not drop the resource state" + ); + assert!( + store.get_resource(&content_commit).await.is_err(), + "ordinary content commits are not stored as resources" + ); + + let mut resource = store.get_resource(&subject).await.unwrap(); + let writer = store.get_default_agent().unwrap().subject.to_string(); + resource + .set(urls::WRITE.into(), vec![writer].into(), &store) + .await + .unwrap(); + let acl = resource.save_locally(&store).await.unwrap(); + let acl_commit = acl.commit_resource.get_subject().clone(); + assert!( + store.get_resource(&acl_commit).await.is_ok(), + "rights-changing commits stay on the must-retain floor" + ); + + let mut resource = store.get_resource(&subject).await.unwrap(); + let destroy = resource.destroy(&store).await.unwrap(); + let destroy_commit = destroy.commit_resource.get_subject().clone(); + assert!( + store.get_resource(&destroy_commit).await.is_ok(), + "destroy commits stay on the must-retain floor" + ); + + // A creation that never set `isGenesis` (Rust `save_locally` on a fresh + // subject, an agent's first commit, an HTTP-subject creation) is still + // the commit that brought the resource into being, and is retained. + let mut unflagged = crate::Resource::new("internal:/unflagged-creation".into()); + unflagged + .set(urls::NAME.into(), Value::String("born".into()), &store) + .await + .unwrap(); + let created = unflagged.save_locally(&store).await.unwrap(); + assert_eq!(created.commit.is_genesis, None, "test premise: no flag"); + assert!( + store + .get_resource(created.commit_resource.get_subject()) + .await + .is_ok(), + "an unflagged creation commit is retained like a genesis" + ); +} diff --git a/lib/src/hierarchy.rs b/lib/src/hierarchy.rs index ba9ba8aae..51b5df502 100644 --- a/lib/src/hierarchy.rs +++ b/lib/src/hierarchy.rs @@ -627,7 +627,6 @@ mod test { validate_signature: false, validate_timestamp: true, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, update_index: true, validate_for_agent: Some(agent.subject.to_string()), @@ -652,7 +651,6 @@ mod test { validate_signature: false, validate_timestamp: true, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, update_index: true, validate_for_agent: Some(agent.subject.to_string()), @@ -712,7 +710,6 @@ mod test { validate_signature: false, validate_timestamp: true, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, update_index: true, validate_for_agent: Some(agent.subject.to_string()), @@ -819,7 +816,6 @@ mod test { validate_signature: false, validate_timestamp: true, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, update_index: true, validate_for_agent: Some(agent.subject.to_string()), diff --git a/lib/src/loro.rs b/lib/src/loro.rs index 9d4b5c6f7..3e905aee2 100644 --- a/lib/src/loro.rs +++ b/lib/src/loro.rs @@ -1998,7 +1998,6 @@ mod test { validate_signature: true, validate_timestamp: true, validate_rights: false, - validate_previous_commit: false, validate_loro_causality: false, update_index: false, validate_for_agent: None, @@ -2043,7 +2042,6 @@ mod test { validate_signature: true, validate_timestamp: true, validate_rights: false, - validate_previous_commit: false, validate_loro_causality: false, update_index: false, validate_for_agent: None, diff --git a/lib/src/parse.rs b/lib/src/parse.rs index 2388cd9dc..714c1d363 100644 --- a/lib/src/parse.rs +++ b/lib/src/parse.rs @@ -859,7 +859,6 @@ async fn parse_json_ad_map_to_resource( validate_signature: true, validate_timestamp: false, validate_rights: parse_opts.for_agent != ForAgent::Sudo, - validate_previous_commit: false, validate_loro_causality: false, validate_for_agent: Some(parse_opts.for_agent.to_string()), update_index: true, @@ -912,7 +911,6 @@ async fn parse_json_ad_map_to_resource( validate_signature: true, validate_timestamp: false, validate_rights: parse_opts.for_agent != ForAgent::Sudo, - validate_previous_commit: false, validate_loro_causality: false, validate_for_agent: Some(parse_opts.for_agent.to_string()), update_index: true, diff --git a/lib/src/populate.rs b/lib/src/populate.rs index 9cdeb83ce..6cef90012 100644 --- a/lib/src/populate.rs +++ b/lib/src/populate.rs @@ -213,13 +213,10 @@ fn base_models() -> (Vec, Vec) { urls::DESTROY.into(), urls::IS_GENESIS.into(), urls::PREVIOUS_COMMIT.into(), - urls::REMOVE.into(), - urls::SET.into(), - urls::PUSH.into(), urls::LORO_UPDATE.into(), ], shortname: "commit".into(), - description: "A Commit is a signed Resource that describes how a Resource must be updated. Used for auditing, versioning and synchronization.".into(), + description: "A signed envelope wrapping a Loro CRDT update. Used to authorize writes. Not a queryable event log — current state lives in the resource's Loro document.".into(), subject: urls::COMMIT.into(), }, ]; diff --git a/lib/src/resources.rs b/lib/src/resources.rs index 3593202f1..8c022cc4c 100644 --- a/lib/src/resources.rs +++ b/lib/src/resources.rs @@ -1103,7 +1103,6 @@ impl Resource { validate_timestamp: false, validate_rights: false, validate_for_agent: Some(agent.subject.to_string()), - validate_previous_commit: false, validate_loro_causality: false, update_index: true, source_id: None, @@ -1222,7 +1221,6 @@ impl Resource { validate_timestamp: false, validate_rights: false, validate_for_agent: Some(agent.subject.to_string()), - validate_previous_commit: false, validate_loro_causality: false, update_index: true, source_id: None, @@ -1257,7 +1255,7 @@ impl Resource { .map(|sig| format!("did:ad:commit:{}", sig)); crate::client::post_commit(&commit, store).await?; self.subject = subject.clone(); - // Store lastCommit so subsequent saves can chain + // Stamp lastCommit so subsequent saves do not mis-detect genesis. if let Some(id) = commit_id { self.propvals .insert(urls::LAST_COMMIT.into(), Value::AtomicUrl(id.into())); @@ -1620,7 +1618,6 @@ mod test { validate_signature: true, validate_timestamp: false, validate_rights: false, - validate_previous_commit: false, validate_loro_causality: false, update_index: true, validate_for_agent: None, @@ -1838,7 +1835,6 @@ mod test { validate_signature: true, validate_timestamp: true, validate_rights: false, - validate_previous_commit: true, validate_loro_causality: false, validate_for_agent: None, update_index: true, diff --git a/lib/src/storelike.rs b/lib/src/storelike.rs index 350e45dc1..c3aec17c3 100644 --- a/lib/src/storelike.rs +++ b/lib/src/storelike.rs @@ -285,7 +285,11 @@ pub trait Storelike: Sized + Send + Sync { ) -> AtomicResult { let applied = commit.validate_and_build_response(opts, self).await?; - self.add_resource(&applied.commit_resource).await?; + // Commits are signed envelopes, not a queryable class. Keep genesis + // and rights/parent/destroy; drop ordinary content certificates. + if applied.auth_impact().is_critical() { + self.add_resource(&applied.commit_resource).await?; + } match (&applied.resource_old, &applied.resource_new) { (None, None) => { diff --git a/lib/src/sync/engine.rs b/lib/src/sync/engine.rs index 1e7c5c8ee..42cc28e21 100644 --- a/lib/src/sync/engine.rs +++ b/lib/src/sync/engine.rs @@ -257,9 +257,8 @@ pub async fn handle_frame_full( .get_base_domain() .unwrap_or_else(|| "http://localhost".to_string()); let subject_resolved = resource.get_subject().resolve(&origin); - // Include `lastCommit` so the recipient can set - // `previousCommit` on its next save. See - // `planning/sync.md` (test coverage gaps, `ws_get`). + // Include `lastCommit` so the recipient can stamp + // `_lastCommit` and not mis-detect genesis on save. let last_commit = resource .get(crate::urls::LAST_COMMIT) .ok() @@ -729,7 +728,6 @@ pub async fn ingest_commit( validate_timestamp: true, validate_rights: true, // https://github.com/atomicdata-dev/atomic-server/issues/412 - validate_previous_commit: false, // Reject commits whose Loro ops are concurrent with stored state // (i.e. the client's doc wasn't seeded from the server). Without this, // LWW silently drops the client's write. For P2P sync use a path that @@ -760,10 +758,9 @@ pub async fn ingest_commit( /// application. It validates signature, schema, and the signer's rights — the /// commit is a self-authorizing certificate, so those checks (not the /// connection's AUTH identity) are the authority. `validate_loro_causality` is -/// off (concurrent peer writes are expected) and `validate_previous_commit` is -/// off (peers don't share a single linear commit chain), mirroring the Iroh -/// sync paths. No `source_id`: peer transports don't fan out through the -/// commit monitor, so there's no echo to suppress. +/// off (concurrent peer writes are expected). No linear `previousCommit` +/// chain: peers merge via Loro. No `source_id`: peer transports don't fan +/// out through the commit monitor, so there's no echo to suppress. /// /// Deliberately skips the server's domain-ownership gate (`apply_commit_json` /// in `server/src/handlers/commit.rs` rejects a commit whose subject belongs diff --git a/lib/src/sync/peer.rs b/lib/src/sync/peer.rs index 16824becd..202a50766 100644 --- a/lib/src/sync/peer.rs +++ b/lib/src/sync/peer.rs @@ -2851,7 +2851,6 @@ mod initiator_trust_tests { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() diff --git a/lib/src/sync/tests.rs b/lib/src/sync/tests.rs index 8f846a065..95b22bf73 100644 --- a/lib/src/sync/tests.rs +++ b/lib/src/sync/tests.rs @@ -618,7 +618,6 @@ mod peer_sync_tests { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() @@ -772,7 +771,6 @@ mod peer_sync_tests { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() @@ -882,7 +880,6 @@ mod peer_sync_tests { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() @@ -1592,7 +1589,6 @@ mod peer_sync_tests { let opts = crate::commit::CommitOpts { validate_signature: true, validate_timestamp: false, - validate_previous_commit: false, validate_rights: false, update_index: true, ..crate::commit::CommitOpts::no_validations_no_index() diff --git a/lib/src/test_utils.rs b/lib/src/test_utils.rs index 932fb6e37..de631ed54 100644 --- a/lib/src/test_utils.rs +++ b/lib/src/test_utils.rs @@ -33,6 +33,10 @@ pub async fn populate_collections(store: &impl Storelike) -> crate::errors::Atom let result = store.query(&query).await?; for subject in result.subjects { + if subject.as_str() == urls::COMMIT { + // Commits are signed envelopes, not a queryable class listing. + continue; + } let mut collection = crate::collections::create_collection_resource_for_class(store, subject.as_str()) .await?; diff --git a/lib/src/values.rs b/lib/src/values.rs index db398e606..b8964b6f6 100644 --- a/lib/src/values.rs +++ b/lib/src/values.rs @@ -360,8 +360,7 @@ impl Value { // TODO: This results in wrong indexing, as some subjects will be numbers. Value::ResourceArray(_v) => self.to_subjects(None).unwrap_or_else(|_| vec![]), Value::AtomicUrl(v) => vec![v.to_string()], - // TODO We don't index nested resources for now - Value::NestedResource(_r) => return None, + Value::NestedResource(_) | Value::LoroDoc(_) => return None, // This might result in unnecessarily long strings, sometimes. We may want to shorten them later. val => vec![val.to_string()], }; @@ -633,6 +632,23 @@ mod test { Value::new(r#"{"en": 5}"#, &DataType::LocalizedText).unwrap_err(); } + #[test] + fn loro_doc_is_not_indexed() { + let blob = vec![0u8; 2048]; + let atom = crate::Atom::new( + "did:ad:commit:test".into(), + crate::urls::LORO_UPDATE.into(), + Value::LoroDoc(blob), + ); + assert!( + atom.to_indexable_atoms().is_empty(), + "Loro binary payloads must not become KV index keys" + ); + assert!(Value::LoroDoc(vec![1, 2, 3]) + .to_reference_index_strings() + .is_none()); + } + #[test] fn value_to_subjects() { let subject_string = String::from("https://example.com/subject_string"); diff --git a/server/src/plugins/wasm.rs b/server/src/plugins/wasm.rs index 2ca916652..0d5f4f802 100644 --- a/server/src/plugins/wasm.rs +++ b/server/src/plugins/wasm.rs @@ -848,7 +848,6 @@ impl bindings::atomic::class_extender::host::Host for PluginHostState { validate_signature: true, validate_timestamp: false, validate_rights: true, - validate_previous_commit: false, validate_loro_causality: false, update_index: true, validate_for_agent: None, From 5ce0b5bf9bbda2cf775e09ed2b7773679a0033fb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 5 Sep 2026 10:33:55 +0000 Subject: [PATCH 2/8] refactor(browser): stop treating discarded commit envelopes as resources UI reads author and dates from the resource. History no longer offers Show Commit, the Sync page does not link commit ids, and sequential saves no longer set previousCommit. Co-authored-by: joepmeindertsma --- .../src/components/CommitDetail.tsx | 83 ++----------------- .../src/components/EditableTitle.tsx | 17 ++-- browser/data-browser/src/locales/de.po | 8 +- browser/data-browser/src/locales/en.po | 8 +- browser/data-browser/src/locales/es.po | 8 +- browser/data-browser/src/locales/fr.po | 8 +- .../src/routes/History/HistoryDesktopView.tsx | 17 ---- browser/data-browser/src/routes/SyncRoute.tsx | 24 ++---- .../src/views/Article/ArticlePage.tsx | 9 +- .../FolderPage/GridItem/ChatRoomGridItem.tsx | 11 +-- .../src/views/FolderPage/ListView.tsx | 14 ++-- .../src/views/ResourcePageDefault.tsx | 12 +-- browser/e2e/tests/e2e.spec.ts | 12 +-- browser/e2e/tests/offline-chatroom.spec.ts | 10 +-- browser/lib/src/commit.test.ts | 35 ++++---- browser/lib/src/commit.ts | 6 +- browser/lib/src/local-only-drive.test.ts | 16 ++-- browser/lib/src/resource.ts | 54 ++++-------- browser/lib/src/store.ts | 48 ++--------- browser/lib/src/test-store.ts | 2 +- 20 files changed, 109 insertions(+), 293 deletions(-) diff --git a/browser/data-browser/src/components/CommitDetail.tsx b/browser/data-browser/src/components/CommitDetail.tsx index 223423064..440feabe8 100644 --- a/browser/data-browser/src/components/CommitDetail.tsx +++ b/browser/data-browser/src/components/CommitDetail.tsx @@ -1,113 +1,48 @@ -import { - commits, - useDate, - useResource, - useString, - type Commits, -} from '@tomic/react'; import { ResourceInline } from '../views/ResourceInline'; import { Detail } from './Detail'; import { DateTime, DateTimeRelative } from './datatypes/DateTime'; -import { AtomicLink } from './AtomicLink'; import type { JSX } from 'react'; type Props = { - commitSubject?: string; short?: boolean; /** - * Date read from an intrinsic resource propval (e.g. a message's - * `createdAt`). When provided it is used as the displayed date — and - * rendered immediately, without waiting on the commit fetch. This is what - * lets the timestamp survive a refresh: under the DID / sign-at-drain model - * a `did:ad:commit:` resource is no longer refetchable, so a date that - * depends on the commit load disappears on reload. See - * `planning/commit-retention-and-state-certificates.md` ("History / audit - * UI"). + * Date read from the resource itself (genesis `createdAt`). No commit + * fetch — `did:ad:commit:` is not a queryable resource. */ createdAt?: Date; /** * Creator (an agent subject) read from the resource itself — e.g. - * `useCreatedBy`, which derives it from the genesis Loro change. Preferred - * over the fetched commit's `signer` so the creator survives a refresh - * without refetching the (no-longer-refetchable) commit. + * `useCreatedBy`, which derives it from the genesis certificate. */ createdBy?: string; }; -/** Shows the latest editor and edit date */ +/** Shows the editor and date from resource-derived metadata. */ export function CommitDetail({ - commitSubject, short, createdAt, createdBy, }: Props): JSX.Element | null { - // Only fetch the commit when the caller hasn't already supplied the creation - // metadata from the resource's own oplog (createdAt + createdBy). Under - // sign-at-drain a `did:ad:commit:` resource is no longer refetchable, so - // when both are provided we skip the fetch entirely — it would only fail and - // (pre-fix) blanked the creator/date on refresh. Other callers (generic - // "last edited" displays) pass neither and still get the commit-fetch path. - const needsCommit = createdAt === undefined || createdBy === undefined; - const resource = useResource( - needsCommit ? commitSubject : undefined, - ); - const [signer] = useString(resource, commits.properties.signer); - const [previousCommit] = useString( - resource, - commits.properties.previousCommit, - ); - - // Prefer the resource-derived creator; fall back to the commit's signer. - const creator = createdBy ?? signer; - - const commitCreatedAt = useDate(resource, commits.properties.createdAt); - const date = createdAt ?? commitCreatedAt; - - if (!commitSubject && !date) { + if (!createdAt) { return null; } - // Wait for a date. With an explicit `createdAt` we render right away (no - // commit needed); otherwise the date comes from the commit, so wait for it. - if (!date || (createdAt === undefined && resource.loading)) { - return -; - } - - // Only link the date to the commit on the legacy commit-fetch path. When the - // metadata is resource-derived (createdAt/createdBy supplied), the commit is - // irrelevant — and under sign-at-drain that link wouldn't resolve — so render - // the date as plain text instead. - const linkToCommit = needsCommit && !!commitSubject; - if (short) { return ( - {linkToCommit ? ( - - - - ) : ( - - )} + ); } - const dateElement = ; + const dateElement = ; return ( - {creator && } + {createdBy && } {'-'} - {linkToCommit ? ( - - {previousCommit ? 'edited ' : ''} - {dateElement} - - ) : ( - dateElement - )}{' '} + {dateElement}{' '} ); } diff --git a/browser/data-browser/src/components/EditableTitle.tsx b/browser/data-browser/src/components/EditableTitle.tsx index 0fe477b80..0e9cf6740 100644 --- a/browser/data-browser/src/components/EditableTitle.tsx +++ b/browser/data-browser/src/components/EditableTitle.tsx @@ -156,16 +156,13 @@ export function EditableTitle({ }, [isEditing, text]); // The keystroke debounce in `useValue` (`commitDebounce: 100ms`) - // means the commit for the last typed character is still parked in - // a `setTimeout` when the user exits the editor. The next interaction - // — back-to-back rename, route change, reload — can run before the - // timer fires, so a quick "type → Escape → type again" sequence ends - // up with the second value chained onto the wrong `previousCommit` - // and the server only keeping the first one. Force-flush on every - // exit path (Enter, Escape, blur) so the commit posts before the - // editor unmounts. `save()` is a no-op when there are no dirty - // changes, so the still-armed debounce timer that fires afterwards - // is harmless. + // means the last typed character is still parked in a `setTimeout` + // when the user exits the editor. The next interaction — back-to-back + // rename, route change, reload — can run before the timer fires and + // drop that last keystroke. Force-flush on every exit path (Enter, + // Escape, blur) so the commit posts before the editor unmounts. + // `save()` is a no-op when there are no dirty changes, so the still- + // armed debounce timer that fires afterwards is harmless. const flushPending = () => { void resource.__internalObject.save().catch(() => undefined); }; diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index 5b5f5cd72..0cb8b87e5 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -1998,8 +1998,8 @@ msgid "Class" msgstr "" #: src/views/FolderPage/ListView.tsx -msgid "Last Modified" -msgstr "" +msgid "Created" +msgstr "Erstellt" #: src/views/Article/ArticlePage.tsx msgid "Children" @@ -2717,10 +2717,6 @@ msgstr "" msgid "Restore this version" msgstr "" -#: src/routes/History/HistoryDesktopView.tsx -msgid "Show Commit" -msgstr "" - #. 0: resource.title #. 0: resource.title #: src/routes/History/HistoryDesktopView.tsx diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index 8e6e2a9e2..df07825cc 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -1998,8 +1998,8 @@ msgid "Class" msgstr "Class" #: src/views/FolderPage/ListView.tsx -msgid "Last Modified" -msgstr "Last Modified" +msgid "Created" +msgstr "Created" #: src/views/Article/ArticlePage.tsx msgid "Children" @@ -2717,10 +2717,6 @@ msgstr "History of" msgid "Restore this version" msgstr "Restore this version" -#: src/routes/History/HistoryDesktopView.tsx -msgid "Show Commit" -msgstr "Show Commit" - #. 0: resource.title #. 0: resource.title #: src/routes/History/HistoryDesktopView.tsx diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index db5ee9c39..a3e0e2953 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -1998,8 +1998,8 @@ msgid "Class" msgstr "Clase" #: src/views/FolderPage/ListView.tsx -msgid "Last Modified" -msgstr "Última modificación" +msgid "Created" +msgstr "Creado" #: src/views/Article/ArticlePage.tsx msgid "Children" @@ -2717,10 +2717,6 @@ msgstr "Historial de" msgid "Restore this version" msgstr "Restaurar esta versión" -#: src/routes/History/HistoryDesktopView.tsx -msgid "Show Commit" -msgstr "Mostrar commit" - #. 0: resource.title #. 0: resource.title #: src/routes/History/HistoryDesktopView.tsx diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index 1afd2c46f..5f9251f5c 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -1998,8 +1998,8 @@ msgid "Class" msgstr "Classe" #: src/views/FolderPage/ListView.tsx -msgid "Last Modified" -msgstr "Dernière modification" +msgid "Created" +msgstr "Créé" #: src/views/Article/ArticlePage.tsx msgid "Children" @@ -2717,10 +2717,6 @@ msgstr "Historique de" msgid "Restore this version" msgstr "Restaurer cette version" -#: src/routes/History/HistoryDesktopView.tsx -msgid "Show Commit" -msgstr "Afficher le Commit" - #. 0: resource.title #. 0: resource.title #: src/routes/History/HistoryDesktopView.tsx diff --git a/browser/data-browser/src/routes/History/HistoryDesktopView.tsx b/browser/data-browser/src/routes/History/HistoryDesktopView.tsx index f23b7bca7..98becf576 100644 --- a/browser/data-browser/src/routes/History/HistoryDesktopView.tsx +++ b/browser/data-browser/src/routes/History/HistoryDesktopView.tsx @@ -7,8 +7,6 @@ import { Column, Row } from '../../components/Row'; import { Title } from '../../components/Title'; import { VersionTitle } from './VersionTitle'; import { VersionScroller } from './VersionScroller'; -import { useNavigateWithTransition } from '../../hooks/useNavigateWithTransition'; -import { constructOpenURL } from '../../helpers/navigation'; import { ResourceDiff, useResourceDiff, @@ -29,7 +27,6 @@ export function HistoryDesktopView({ onSelectVersion, onVersionAccept, }: HistoryViewProps) { - const navigate = useNavigateWithTransition(); const store = useStore(); const selectedVersionResource = useMemo(() => { @@ -64,10 +61,6 @@ export function HistoryDesktopView({ { label: 'Resource', value: 'resource' }, ]; - const lastCommit = selectedVersion.propvals.get( - 'https://atomicdata.dev/properties/lastCommit', - ) as string | undefined; - return ( <> @@ -91,16 +84,6 @@ export function HistoryDesktopView({ - diff --git a/browser/data-browser/src/routes/SyncRoute.tsx b/browser/data-browser/src/routes/SyncRoute.tsx index 845fe780e..6f7753367 100644 --- a/browser/data-browser/src/routes/SyncRoute.tsx +++ b/browser/data-browser/src/routes/SyncRoute.tsx @@ -1941,22 +1941,14 @@ function SyncPage() { {entry.destroy && destroy} - {entry.commitId ? ( - - - {formatTimeAgo(new Date(entry.timestamp)) ?? - 'just now'} - - - ) : ( - - {formatTimeAgo(new Date(entry.timestamp)) ?? 'just now'} - - )} + {/* `entry.commitId` is a `did:ad:commit:` receipt, not a + fetchable resource (content commits are not stored), so + the timestamp is plain text. */} + + {formatTimeAgo(new Date(entry.timestamp)) ?? 'just now'} + diff --git a/browser/data-browser/src/views/Article/ArticlePage.tsx b/browser/data-browser/src/views/Article/ArticlePage.tsx index cc39b96bf..fab01c7ed 100644 --- a/browser/data-browser/src/views/Article/ArticlePage.tsx +++ b/browser/data-browser/src/views/Article/ArticlePage.tsx @@ -1,9 +1,9 @@ import { - commits, dataBrowser, useCanWrite, useChildren, - useString, + useCreatedBy, + useCreatedAt, } from '@tomic/react'; import { useCallback, type JSX } from 'react'; import { styled } from 'styled-components'; @@ -20,7 +20,8 @@ import { ArticleDescription } from './ArticleDescription'; import { useNewResourceUI } from '../../components/forms/NewForm/useNewResourceUI'; export function ArticlePage({ resource }: ResourcePageProps): JSX.Element { - const [lastCommit] = useString(resource, commits.properties.lastCommit); + const createdBy = useCreatedBy(resource); + const createdAt = useCreatedAt(resource); const canEdit = useCanWrite(resource); const { subjects: children } = useChildren(resource.subject); @@ -40,7 +41,7 @@ export function ArticlePage({ resource }: ResourcePageProps): JSX.Element { - + diff --git a/browser/data-browser/src/views/FolderPage/GridItem/ChatRoomGridItem.tsx b/browser/data-browser/src/views/FolderPage/GridItem/ChatRoomGridItem.tsx index 4a5000ec2..b2e3a1d88 100644 --- a/browser/data-browser/src/views/FolderPage/GridItem/ChatRoomGridItem.tsx +++ b/browser/data-browser/src/views/FolderPage/GridItem/ChatRoomGridItem.tsx @@ -1,9 +1,9 @@ import { properties, useArray, + useCreatedBy, useResource, useString, - useSubject, useTitle, } from '@tomic/react'; @@ -39,13 +39,8 @@ interface LastMessageProps { const Message = ({ subject, alignment }: LastMessageProps): JSX.Element => { const messageResource = useResource(subject); - const [lastCommit] = useSubject( - messageResource, - properties.commit.lastCommit, - ); - const lastCommitResource = useResource(lastCommit); - const [signer] = useSubject(lastCommitResource, properties.commit.signer); - const signerResource = useResource(signer); + const createdBy = useCreatedBy(messageResource); + const signerResource = useResource(createdBy); const [signerName] = useTitle(signerResource); const [text] = useString(messageResource, properties.description); diff --git a/browser/data-browser/src/views/FolderPage/ListView.tsx b/browser/data-browser/src/views/FolderPage/ListView.tsx index f825c4b4a..8e1e5eac9 100644 --- a/browser/data-browser/src/views/FolderPage/ListView.tsx +++ b/browser/data-browser/src/views/FolderPage/ListView.tsx @@ -1,7 +1,8 @@ import { - commits, core, Resource, + useCreatedBy, + useCreatedAt, useResource, useString, useTitle, @@ -33,7 +34,7 @@ export function ListView({ Title Class - {!basic && Last Modified} + {!basic && Created} @@ -47,7 +48,7 @@ export function ListView({ {!basic && ( - + )} @@ -84,12 +85,13 @@ function Title({ resource }: CellProps): JSX.Element { ); } -function LastCommit({ resource }: CellProps): JSX.Element { - const [commit] = useString(resource, commits.properties.lastCommit); +function Created({ resource }: CellProps): JSX.Element { + const createdAt = useCreatedAt(resource); + const createdBy = useCreatedBy(resource); return ( - + ); } diff --git a/browser/data-browser/src/views/ResourcePageDefault.tsx b/browser/data-browser/src/views/ResourcePageDefault.tsx index 5b77a2a83..f5418b930 100644 --- a/browser/data-browser/src/views/ResourcePageDefault.tsx +++ b/browser/data-browser/src/views/ResourcePageDefault.tsx @@ -1,10 +1,11 @@ import { - useString, + useCreatedBy, + useCreatedAt, + useCanWrite, + dataBrowser, core, server, commits, - useCanWrite, - dataBrowser, } from '@tomic/react'; import { styled } from 'styled-components'; import AllProps from '../components/AllProps'; @@ -59,7 +60,8 @@ export const defaultHiddenProps = [ export function ResourcePageDefault({ resource, }: ResourcePageProps): JSX.Element { - const [lastCommit] = useString(resource, commits.properties.lastCommit); + const createdBy = useCreatedBy(resource); + const createdAt = useCreatedAt(resource); const canEdit = useCanWrite(resource); const navigate = useNavigateWithTransition(); @@ -81,7 +83,7 @@ export function ResourcePageDefault({ - + { ).toBeVisible({ timeout: 15_000 }); // The message's CommitDetail row should show the author's name AND the - // date. The author name comes from the agent resource's `name` propval - // — `devDrive()` sets it to "Dev User" — and only renders if the - // commit was persisted server-side and the signer resource is - // loadable back. The date comes from the commit's `createdAt`. Both - // together are a tight roundtrip check: client signed → server - // stored → refetched + rendered by ``. Scope to the + // date. Both come from the message resource (genesis certificate), not + // from fetching a `did:ad:commit:` envelope. Scope to the // message element (the styled
wrapping the message body + // CommitDetail; it carries `about={subject}` in the DOM) by walking // up from the message paragraph, so we don't accidentally match @@ -218,7 +214,7 @@ test.describe('data-browser', async () => { await expect(messageLocator).toBeVisible(); await expect( messageLocator, - 'Message author "Dev User" missing — commit author not stored/retrievable', + 'Message author "Dev User" missing — genesis createdBy not retrievable', ).toContainText('Dev User'); // The visible label is intentionally relative ("now", "1 minute ago"). // Assert the semantic timestamp instead so the test proves `createdAt` @@ -226,7 +222,7 @@ test.describe('data-browser', async () => { const year = new Date().getFullYear().toString(); await expect( messageLocator.locator('time'), - 'Message date missing — commit createdAt not stored/retrievable', + 'Message date missing — genesis createdAt not retrievable', ).toHaveAttribute('datetime', new RegExp(`^${year}-`)); // Regression: author + date must SURVIVE A REFRESH. They are derived from diff --git a/browser/e2e/tests/offline-chatroom.spec.ts b/browser/e2e/tests/offline-chatroom.spec.ts index 856208662..935f78c53 100644 --- a/browser/e2e/tests/offline-chatroom.spec.ts +++ b/browser/e2e/tests/offline-chatroom.spec.ts @@ -175,13 +175,9 @@ test.describe('offline chatroom', () => { } // Correct order: walk the DOM and collect the FIRST text occurrence of - // each expected message in document order. We can't just iterate all - // `[about^="did:ad:"]` wrappers — `` inside each message - // also has an `about=` attribute, and a related-but- - // separate hydration bug fills the commit's propvals with the - // committed-resource's data, so a naive walk would count those copies. - // Using `getByText(m).first()` gives us the message's own - // `` rendering, which sits exactly once per message. + // each expected message in document order. Using `getByText(m).first()` + // gives us the message's own `` rendering, which sits + // exactly once per message. const positions: Array<{ text: string; y: number }> = []; for (const text of MESSAGES) { diff --git a/browser/lib/src/commit.test.ts b/browser/lib/src/commit.test.ts index 83479bd65..b316ecddf 100644 --- a/browser/lib/src/commit.test.ts +++ b/browser/lib/src/commit.test.ts @@ -18,7 +18,7 @@ import { testStore } from './test-store.js'; * server has acked. */ describe('Resource save flow', () => { - it('creates a DID resource and chains commits on sequential saves', async ({ + it('creates a DID resource and posts sequential saves', async ({ expect, }) => { const { store, postCommitSpy } = await testStore(); @@ -35,8 +35,8 @@ describe('Resource save flow', () => { expect(await doc.save()).toBe('persisted'); - // A remote merge that drops `lastCommit` must not break chaining — - // the resource keeps its own commit cursor. + // A remote merge that drops `lastCommit` must not break the next + // save — the resource keeps its own commit cursor. doc.removeUnsafe('https://atomicdata.dev/properties/lastCommit'); await doc.set( @@ -49,12 +49,15 @@ describe('Resource save flow', () => { // Subject is stable across saves. expect(doc.subject).toBe(genesisSubject); - // Two commits, the second chained on the first. + // Two commits: genesis then an update. No previousCommit chain. expect(postCommitSpy.mock.calls.length).toBe(2); const first = postCommitSpy.mock.calls[0][0] as Commit; const second = postCommitSpy.mock.calls[1][0] as Commit; + expect(first.isGenesis).toBe(true); + expect(first.previousCommit).toBeUndefined(); expect(second.subject).toBe(genesisSubject); - expect(second.previousCommit).toContain(first.signature!); + expect(second.isGenesis).toBeFalsy(); + expect(second.previousCommit).toBeUndefined(); }); it('supports typed property setters via the props proxy', async ({ @@ -193,16 +196,16 @@ describe('Resource save flow', () => { ); }); - it('caches the just-saved commit locally so needs no fetch', async ({ + it('does not cache the just-saved commit as a store resource', async ({ expect, }) => { /** - * Regression: a chatroom message post used to trigger a - * `GET did:ad:commit:` because the commit wasn't materialized - * locally. After `save()`, the commit's DID subject must be present - * in `store.resources` with the propvals reads. + * Commits are signed envelopes, not queryable resources. + * After `save()`, the commit DID must not appear in `store.resources` + * — UI reads author/date from the resource, not by fetching + * `did:ad:commit:…`. */ - const { store, posted, agentDID } = await testStore(); + const { store, posted } = await testStore(); const msg = await store.newResource({ isA: 'https://atomicdata.dev/classes/Message', @@ -213,15 +216,7 @@ describe('Resource save flow', () => { expect(posted.length).toBe(1); const commitDidSubject = `did:ad:commit:${posted[0].signature}`; - expect(store.resources.has(commitDidSubject)).toBe(true); - - const commitResource = store.resources.get(commitDidSubject)!; - expect(commitResource.get('https://atomicdata.dev/properties/signer')).toBe( - agentDID, - ); - expect( - commitResource.get('https://atomicdata.dev/properties/createdAt'), - ).toBeTypeOf('number'); + expect(store.resources.has(commitDidSubject)).toBe(false); }); }); diff --git a/browser/lib/src/commit.ts b/browser/lib/src/commit.ts index ee6dc2e0a..4039fb112 100644 --- a/browser/lib/src/commit.ts +++ b/browser/lib/src/commit.ts @@ -29,8 +29,7 @@ export interface CommitBuilderI { /** If true, the resource must be deleted. https://atomicdata.dev/properties/destroy */ destroy?: boolean; /** - * URL of the previous Commit, used by the receiver to make sure that we're - * having the same current version. + * URL of an earlier envelope, optional audit metadata. Not a causal gate. */ previousCommit?: string; /** Whether this is the first commit for a Resource. */ @@ -134,8 +133,7 @@ export class CommitBuilder { } /** - * Set the URL of the Commit that was previously (last) applied. The value of - * this should probably be the `lastCommit` of the Resource. + * Optional audit pointer at an earlier envelope. Not a causal gate. */ public setPreviousCommit(prev: string): CommitBuilder { this._previousCommit = prev; diff --git a/browser/lib/src/local-only-drive.test.ts b/browser/lib/src/local-only-drive.test.ts index 95ad10333..ddff16d63 100644 --- a/browser/lib/src/local-only-drive.test.ts +++ b/browser/lib/src/local-only-drive.test.ts @@ -3,9 +3,9 @@ import { core, commits, server } from './index.js'; import { testStore } from './test-store.js'; /** - * Local-only drives (e.g. the demo workspace): resources save, chain - * commits, and materialize history entirely client-side — nothing may - * ever be POSTed or enrolled in the outbox. + * Local-only drives (e.g. the demo workspace): resources save and + * stamp lastCommit entirely client-side — nothing may ever be POSTed + * or enrolled in the outbox. */ describe('Local-only drives', () => { it('saves the drive without POSTing or enrolling the outbox', async ({ @@ -27,13 +27,12 @@ describe('Local-only drives', () => { expect(store.outbox.size).toBe(0); expect(drive.subject.startsWith('did:ad:')).toBe(true); - // The genesis commit is materialized locally for the history log. const lastCommit = String(drive.get(commits.properties.lastCommit)); expect(lastCommit.startsWith('did:ad:commit:')).toBe(true); - expect(store.resources.has(lastCommit)).toBe(true); + expect(store.resources.has(lastCommit)).toBe(false); }); - it('resolves children as local-only and chains their commits locally', async ({ + it('resolves children as local-only and stamps lastCommit locally', async ({ expect, }) => { const { store, posted } = await testStore(); @@ -63,11 +62,6 @@ describe('Local-only drives', () => { expect(second).not.toBe(first); - // The edit's commit chains on the genesis, exactly like the drain - // would have chained it. - const editCommit = store.resources.get(second); - expect(editCommit?.get(commits.properties.previousCommit)).toBe(first); - expect(posted).toHaveLength(0); expect(store.outbox.size).toBe(0); }); diff --git a/browser/lib/src/resource.ts b/browser/lib/src/resource.ts index ac02e0aa7..42199d248 100644 --- a/browser/lib/src/resource.ts +++ b/browser/lib/src/resource.ts @@ -213,8 +213,9 @@ export class Resource { private _loroVersionAtLastSave?: VersionVector; /** - * The subject of the most recently applied commit. This is the source of truth - * for the commit chain and is protected from being clobbered by remote merges. + * The id of the most recently applied commit (`did:ad:commit:{sig}`). + * Stamp for genesis detection and echo-dedup; not a fetchable resource. + * Protected from being clobbered by remote merges. */ private _lastCommit: string | undefined; @@ -1143,9 +1144,8 @@ export class Resource { ); } - /** The canonical `previousCommit` value for chaining the next sign: - * the in-memory `_lastCommit` (set by `setLastCommitValue` after - * every ack) falls back to the cached property. + /** The resource's last applied commit id (`did:ad:commit:{sig}`). + * Stamp for genesis detection and echo-dedup — not a fetchable resource. * @internal store-level drain only — not part of the public API. */ public getLastCommitForChain(): string | undefined { return ( @@ -2305,19 +2305,12 @@ export class Resource { const newCommitBuilder = new CommitBuilder(this.subject); newCommitBuilder.setDestroy(true); - // The server rejects destroy commits without `previousCommit` for - // non-genesis resources. If a fetch is in flight, wait for it. + // If a fetch is in flight, wait for it so we destroy the current + // resource rather than a stale in-memory copy. if (this.loading) { await this.store.getResource(this.subject).catch(() => undefined); } - const lastCommit = - this._lastCommit ?? this.get(properties.commit.lastCommit)?.toString(); - - if (lastCommit) { - newCommitBuilder.setPreviousCommit(lastCommit); - } - if (agent === undefined) { agent = this.store.getAgent(); } @@ -2657,9 +2650,7 @@ export class Resource { /** * Sign pending changes into a {@link Commit}. * - * - For DID genesis commits the subject is replaced with `did:ad:`. - * - Locally-queued commits are chained via their signatures so that - * `previousCommit` stays consistent even before pushing. + * For DID genesis commits the subject is replaced with `did:ad:`. * * @returns The signed {@link Commit}. * @@ -2689,11 +2680,9 @@ export class Resource { this.rebuildCacheFromLoro(); this.#cacheDirty = false; - // Chain on the resource's lastCommit (server-acked). Under + // Genesis detection uses the lastCommit stamp (server-acked). Under // sign-at-drain there's at most one signed-but-unposted commit - // per subject (the optional `signedGenesis` in the outbox), and - // genesis commits don't have a previousCommit, so we never need - // to chain on an unposted local commit. + // per subject (the optional `signedGenesis` in the outbox). const lastCommit = this._lastCommit ?? this.get(properties.commit.lastCommit)?.toString(); const isFirstCommit = !lastCommit; @@ -2811,10 +2800,6 @@ export class Resource { : undefined, ); - if (lastCommit) { - this.#commitBuilder.setPreviousCommit(lastCommit); - } - // Export Loro delta — the sole carrier of property changes. Pass the agent // again as a fallback: if `writeDatatypeTags` had nothing to commit, this // export's commit is the one that creates the genesis change. @@ -2832,9 +2817,10 @@ export class Resource { this.#commitBuilder.setLoroUpdate(loroDelta); } - // Auto-detect genesis: no previousCommit means this is a new resource. - // The server requires is_genesis=true for DID resources without a previous commit. - // Only for DID-eligible subjects (_new: or did:ad:) — HTTP URLs use server-assigned subjects. + // Auto-detect genesis: no lastCommit stamp means this is a new resource. + // The server requires is_genesis=true for DID resources that do not + // yet exist. Only for DID-eligible subjects (_new: or did:ad:) — + // HTTP URLs use server-assigned subjects. // Agents are excluded: their identity is fixed by their public key // (did:ad:agent:), not derived from a genesis-commit signature. // Marking an agent commit as genesis triggers `delete subject` below, @@ -2847,7 +2833,7 @@ export class Resource { this.subject.startsWith('_new:') || this.subject.startsWith('did:ad:'); const isAgent = this.subject.startsWith('did:ad:agent:'); - if (isDIDEligible && !isAgent && !this.#commitBuilder.previousCommit) { + if (isDIDEligible && !isAgent && isFirstCommit) { this.#commitBuilder.setIsGenesis(true); } @@ -3112,7 +3098,7 @@ export class Resource { // `set()` + `save()` with no explicit genesis step. The real // `did:ad:` // subject only exists after signing, so sign now: `signChanges` - // auto-detects genesis (no previousCommit + DID-eligible), derives + // auto-detects genesis (no lastCommit stamp + DID-eligible), derives // the DID, and renames this resource in place; we enqueue the // signed genesis under the NEW subject. Without this a `_new:` // subject would be marked dirty and the drain would POST a commit @@ -3225,8 +3211,8 @@ export class Resource { if (genesis) { settled.push(genesis); - // The delta sign below chains on `lastCommit` — point it at the - // genesis first. + // The delta sign below uses lastCommit as the "already exists" + // stamp — point it at the genesis first. this.setLastCommitValue(`did:ad:commit:${genesis.signature}`); } @@ -3249,7 +3235,6 @@ export class Resource { } for (const commit of settled) { - this.store.materializeCommitLocally(commit); this.setLastCommitValue(`did:ad:commit:${commit.signature}`); this.store.logLocalOnlyCommitSettled(commit); } @@ -3270,8 +3255,6 @@ export class Resource { * envelope) but no incremental signed commits — those are signed * fresh from the Loro delta at drain time. So the offline path: * - * - Materializes the pre-signed genesis (if present) as a - * `CommitDetail`-renderable resource for the offline audit log. * - Persists this resource atomically (JSON-AD + Loro snapshot) to * clientDb so a reload can hydrate the Loro state before the WS * reconnect drain re-signs from the same `_loroVersionAtLastSave` @@ -3288,7 +3271,6 @@ export class Resource { )?.signedGenesis; if (signedGenesis) { - this.store.materializeCommitLocally(signedGenesis); this.setLastCommitValue(`did:ad:commit:${signedGenesis.signature}`); } diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index f24e59981..14c4bb384 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -9,7 +9,6 @@ import { Client, type FileOrFileLike } from './client.js'; import { CommitBuilder, commitIdOf, - commitToJsonADObject, type Commit, } from './commit.js'; import { datatypeFromUrl, type Datatype } from './datatypes.js'; @@ -1120,7 +1119,7 @@ export class Store { * from this version. * 2. If the subject has accumulated local Loro ops (`markDirty` was * called since the last successful drain), export the delta, - * sign ONE commit chained on `resource.lastCommit`, POST. On + * sign ONE commit, POST. On * success: clear dirty, `setLastCommitValue`, advance cursor. * * Resource must be loaded in the store; cold drains for unloaded @@ -1306,8 +1305,8 @@ export class Store { resource.restoreSaveCursor(entry.baseVersion); } - const previousCommit = resource.getLastCommitForChain(); - const isFirstCommit = !previousCommit; + const lastCommit = resource.getLastCommitForChain(); + const isFirstCommit = !lastCommit; // Tag this commit's Loro change with a unique token so the oplog keeps // a distinct Change per Atomic commit — `getLoroHistory` buckets by it // to reconstruct one version per commit. The token only needs to be @@ -1351,7 +1350,6 @@ export class Store { const { bytes: delta, versionAfterExport } = exported; const builder = new CommitBuilder(subject); - if (previousCommit) builder.setPreviousCommit(previousCommit); builder.setLoroUpdate(delta); const commit = await builder.sign(agent); @@ -4998,7 +4996,7 @@ export class Store { } /** @internal Settle a commit that will never be POSTed (local-only - * drives sign and materialize locally). Transitions the `pending` + * drives sign locally). Transitions the `pending` * entry `logPendingCommit` created so the Sync page doesn't show it * as queued forever. */ public logLocalOnlyCommitSettled(commit: Commit): void { @@ -5010,7 +5008,7 @@ export class Store { if (commit.destroy) { parts.push('destroy'); - } else if (!commit.previousCommit) { + } else if (commit.isGenesis) { parts.push('created'); } else { parts.push('updated'); @@ -5316,7 +5314,7 @@ export class Store { /** Posts a Commit to some endpoint. Returns the Commit created by the server. */ public async postCommit(commit: Commit, endpoint: string): Promise { const close = perfSpan('store.postCommit', { - genesis: commit.previousCommit === undefined, + genesis: !!commit.isGenesis, }); try { @@ -5327,14 +5325,6 @@ export class Store { commitId: commitIdOf(created), }), ); - // Materialize the just-signed commit as a Resource so subsequent - // `useResource(commitSubject)` lookups (chatroom , - // version views, etc.) hit the local cache instead of round- - // tripping back to the server for data we already had in hand. - // The local-only save branch already does this; the online happy - // path used to skip it, which produced the `GET did:ad:commit:*` - // visible in the network log right after posting a chat message. - this.materializeCommitLocally(created); return created; } catch (e) { @@ -5387,32 +5377,6 @@ export class Store { } } - /** - * Cache a freshly-signed commit as a Resource in the local store. - * Idempotent: bails if the commit's subject is already present - * (e.g. the local-only save path beat us to it). - */ - public materializeCommitLocally(commit: Commit): void { - const signature = commit.signature; - if (!signature) return; - const commitSubject = `did:ad:commit:${signature}`; - if (this.resources.has(commitSubject)) return; - - const commitResource = new Resource(commitSubject); - commitResource.applyHydratedValues( - Object.entries(commitToJsonADObject(commit)) as Iterable< - [string, any] // eslint-disable-line @typescript-eslint/no-explicit-any - >, - ); - commitResource.loading = false; - commitResource.new = false; - this.applyIncoming({ - subject: commitSubject, - resource: commitResource, - source: 'local-post', - }); - } - /** * Returns the ancestry of a resource, starting with the resource itself. */ diff --git a/browser/lib/src/test-store.ts b/browser/lib/src/test-store.ts index a1504b91f..84ad57644 100644 --- a/browser/lib/src/test-store.ts +++ b/browser/lib/src/test-store.ts @@ -8,7 +8,7 @@ export interface TestStore { store: Store; agentDID: string; /** Every commit handed to `client.postCommit`, in order. The signed - * envelope the server would receive — assert `previousCommit`, + * envelope the server would receive — assert `isGenesis`, * `loroUpdate`, `subject`, count, etc. against these. */ posted: Commit[]; /** `client.postCommit` spy (echoes the commit back with an `id`). */ From 8d33844cc2cfc9bfaf3a76d13528f10fbf7c9c1f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 5 Sep 2026 10:33:59 +0000 Subject: [PATCH 3/8] docs: describe commits as signed envelopes, not an event log Align public docs, README, changelogs, and test-coverage notes with apply-and-discard for ordinary content commits. Co-authored-by: joepmeindertsma --- CHANGELOG.md | 9 ++- README.md | 2 +- TESTING_COVERAGE.md | 8 +++ browser/CHANGELOG.md | 4 ++ docs/src/atomic-server.md | 2 +- docs/src/commits/concepts.md | 9 +-- docs/src/commits/intro.md | 68 ++++++++++----------- docs/src/interoperability/graph-database.md | 4 +- docs/src/interoperability/rdf.md | 6 +- docs/src/interoperability/solid.md | 20 +++--- docs/src/interoperability/sql.md | 2 +- docs/src/roadmap.md | 2 +- docs/src/websockets.md | 2 + 13 files changed, 76 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acd87f72e..f09bdb0d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,14 @@ By far most changes relate to `atomic-server`, so if not specified, assume the c **Changes to JS assets (including the front-end and JS libraries) are not shown here**, but in [`/browser/CHANGELOG`](/browser/CHANGELOG.md). See [STATUS.md](server/STATUS.md) to learn more about which features will remain stable. -## UNRELEASED +- Commits are signed envelopes, not a queryable event log. Ordinary content + commits are not stored as resources after apply (genesis and + rights/parent/destroy stay). Loro binaries are not KV-index keys. The + `/commits` collection is not created. UI reads author/dates from the + resource, not by fetching `did:ad:commit:` rows. A creation commit is + retained whether or not the client flagged `isGenesis`. Clients no longer chain + `previousCommit`; apply no longer has a previous-commit validation gate. + History no longer offers a "Show Commit" link at a discarded envelope. - **Missing-drive bootstrap is no longer a free pass (OQ5).** A `SYNC_PUSH` or live write for a drive this node has never stored goes diff --git a/README.md b/README.md index fd174cd52..de86ffaa5 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ _Status: alpha. [Breaking changes](CHANGELOG.md) are expected until 1.0._ - 📄 **Documents**, collaborative, rich text, similar to Google Docs / Notion. - 💬 **Group chat**, performant and flexible message channels with attachments, search and replies. - 📂 **File management**: Upload, download and preview attachments. -- 💾 **Event-sourced versioning** / history powered by [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) +- 💾 **Versioning** / history from the Loro oplog, with writes authorized by [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) - 🔄 **Real-time synchronization**: instantly communicates state changes with a client. Build dynamic, collaborative apps using [websockets](https://docs.atomicdata.dev/websockets) (using a [single one-liner in react](https://docs.atomicdata.dev/usecases/react) or [svelte](https://docs.atomicdata.dev/svelte)). - 🧰 **Many serialization options**: to JSON, [JSON-AD](https://docs.atomicdata.dev/core/json-ad.html), and various Linked Data / RDF formats (RDF/XML, N-Triples / Turtle / JSON-LD). - 📖 **Pagination, sorting and filtering** queries using [Atomic Collections](https://docs.atomicdata.dev/schema/collections.html). diff --git a/TESTING_COVERAGE.md b/TESTING_COVERAGE.md index 304c9e065..6b6a7cd78 100644 --- a/TESTING_COVERAGE.md +++ b/TESTING_COVERAGE.md @@ -348,6 +348,14 @@ Not covered: derived AI tools invoked through a real model; MCP protocol project Not covered: leftover Yjs-era DocumentV2 bodies end-to-end (needs a stored `{ type: 'ydoc' }` fixture); read-only v1 documents stay on the element list and have no e2e. +## Commits as envelopes + +| Flow | Layer | Where | +|---|---|---| +| `LoroDoc` values are not KV-index keys | protocol | `lib/src/values.rs::loro_doc_is_not_indexed` | +| Content commits are not stored; genesis/ACL/destroy are | protocol | `lib/src/db/test.rs::content_commits_are_not_stored` | +| Sequential saves do not chain `previousCommit`; commit DIDs are not store resources | glue | `browser/lib/src/commit.test.ts` | + ## Personal drive identity | Flow | Where | diff --git a/browser/CHANGELOG.md b/browser/CHANGELOG.md index e32033e48..b584ae62f 100644 --- a/browser/CHANGELOG.md +++ b/browser/CHANGELOG.md @@ -4,6 +4,10 @@ This changelog covers all five packages, as they are (for now) updated as a whol ## UNRELEASED +- Commits are signed envelopes: `CommitDetail` does not fetch `did:ad:commit:` + resources; author and dates come from the resource. History no longer + navigates to a commit DID, and the Sync page log no longer links commit ids. + Sequential saves no longer set `previousCommit`. - [#1232](https://github.com/ontola/atomic-server/issues/1232) Unified actions: the ⌘K palette shows a capped Actions section for the current resource, hotkeys and the shortcuts overlay/page render from the action diff --git a/docs/src/atomic-server.md b/docs/src/atomic-server.md index f8555f9f2..b2da3afc4 100644 --- a/docs/src/atomic-server.md +++ b/docs/src/atomic-server.md @@ -22,7 +22,7 @@ It's free, open source (MIT license), and has a ton of features: - 📄 **Documents**, collaborative, rich text, similar to Google Docs / Notion. - 💬 **Group chat**, performant and flexible message channels with attachments, search and replies. - 📂 **File management**: Upload, download and preview attachments. -- 💾 **Event-sourced versioning** / history powered by [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) +- 💾 **Versioning** / history from the Loro oplog, with writes authorized by [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) - 🔄 **Real-time synchronization**: instantly communicates state changes with a client. Build dynamic, collaborative apps using [websockets](https://docs.atomicdata.dev/websockets) (using a [single one-liner in react](https://docs.atomicdata.dev/usecases/react) or [svelte](https://docs.atomicdata.dev/svelte)). - 🧰 **Many serialization options**: to JSON, [JSON-AD](https://docs.atomicdata.dev/core/json-ad.html), and various Linked Data / RDF formats (RDF/XML, N-Triples / Turtle / JSON-LD). - 📖 **Pagination, sorting and filtering** queries using [Atomic Collections](https://docs.atomicdata.dev/schema/collections.html). diff --git a/docs/src/commits/concepts.md b/docs/src/commits/concepts.md index 04726656c..777c1f391 100644 --- a/docs/src/commits/concepts.md +++ b/docs/src/commits/concepts.md @@ -5,9 +5,10 @@ _url: [https://atomicdata.dev/classes/Commit](https://atomicdata.dev/classes/Commit)_ -A Commit is a Resource that describes how a Resource must be updated. -It can be used for auditing, versioning and feeds. -It is cryptographically signed by an [Agent](https://atomicdata.dev/classes/Agent). +A Commit is a signed envelope that authorizes a Loro CRDT update on a Resource. +It is **not** a queryable event log. Current state and version history live in +the Resource's Loro document. The commit may be discarded after apply, except +for the must-retain floor (genesis, rights / parent / destroy). All state changes in Atomic Data are carried as [Loro CRDT](https://loro.dev) binary updates. This means that concurrent edits from multiple clients merge automatically without conflicts. @@ -23,7 +24,7 @@ The **optional fields** are: - `loroUpdate` - A [Loro CRDT](https://loro.dev) binary update, encoded as a base64 string. This is the primary way to carry property changes. The server imports this update into the resource's Loro document, materializes the properties, and computes index diffs. - `destroy` - If true, the entire Resource will be removed. -- `previousCommit` - The `did:ad:commit:{signature}` of the last commit applied to this resource. Used for ordering and audit trails. +- `previousCommit` - Optional audit pointer at an earlier envelope. **Not a causal gate** — concurrent edits merge via Loro. Clients may still send it; servers do not require it. - `isGenesis` - If true, this is the first commit for a DID resource. The subject DID is derived from either the self-verifying genesis certificate signature (`did:ad:`) or, in legacy mode, the signature of the genesis commit itself. ### Loro CRDT updates diff --git a/docs/src/commits/intro.md b/docs/src/commits/intro.md index b9ee58cbb..08abe4f3d 100644 --- a/docs/src/commits/intro.md +++ b/docs/src/commits/intro.md @@ -1,50 +1,44 @@ -{{#title Atomic Commits - Event standard for Atomic Data}} +{{#title Atomic Commits - Signed write envelopes}} # Atomic Commits -_Disclaimer: Work in progress, prone to change._ +Atomic Commits are **signed envelopes** that authorize a write to a Resource. +They are not an event-sourced history, and they are not a queryable class of +resources. Current state lives in the Resource's [Loro CRDT](https://loro.dev) +document. History and versioning read that document's oplog. A commit is the +small signed receipt the transport is not allowed to omit. -Atomic Commits is a specification for communicating _state changes_ (events / transactions / patches / deltas / mutations) of [Atomic Data](../core/concepts.md). -It is the part of Atomic Data that is concerned with writing, editing, removing and updating information. +See [concepts](concepts.md) for the field list. ## Design goals -- **Event sourced**: Store and standardize _changes_, as well as the _current_ state. This enables versioning, history playback, undo, audit logs, and more. -- **Traceable origin**: Every change should be traceable to an actor and a point in time. -- **Verifiable**: Have cryptographic proof for every change. Know _when_, and _what_ was changed by _whom_. -- **Identifiable**: A single commit has an identifier - it is a resource. -- **Decentralized**: Commits can be shared in P2P networks from device to device, whilst maintaining verifiability. -- **Extensible**: The methods inside a commit are not fixed. Use-case specific methods can be added by anyone. -- **Streamable**: The commits could be used in streaming context. -- **Familiar**: Introduces as little new stuff as possible (no new formats or language to learn) -- **Pub/Sub**: Subscribe to changes and get notified on changes. -- **ACID-compliant**: An Atomic commit will only occur if it results in a valid state. -- **Atomic**: All the Atomic Data design goals also apply here. +- **Verifiable writes**: cryptographic proof of who changed what, and when. +- **Traceable origin**: every applied write is attributable to an Agent. +- **CRDT merge**: concurrent edits merge deterministically via Loro. There is no linear commit chain to enforce. +- **Identifiable**: a commit has an id (`did:ad:commit:{signature}`). That id may be retained as a receipt; it is not required as a refetchable resource. +- **Decentralized**: envelopes can move over HTTP `/commit`, WebSocket `COMMIT`, or a peer sync path that carries the same signature. +- **ACID-compliant**: a commit is applied only if signature, rights, and schema checks pass. +- **Atomic**: all Atomic Data design goals also apply here. -## Motivation - -Although it's a good idea to keep data at the source as much as possible, we'll often need to synchronize two systems. -For example when data has to be queried or indexed differently than its source can support. -Doing this synchronization can be very difficult, since most of our software is designed to only maintain and share the _current state_ of a system. +## What a commit is not -I noticed this mainly when working on OpenBesluitvorming.nl - an open data project where we aimed to fetch and standardize meeting data (votes, meeting minutes, documents) from 150+ local governments in the Netherlands. -We wrote software that fetched data from various systems (who all had different models, serialization formats and APIs), transformed this data to a single standard and share it through an API and a fulltext search endpoint. -One of the hard parts was keeping our data in sync with the sources. -How could we now if something was changed upstream? -We queried all these systems every night for _all meetings from the next and previous month_, and made deep comparisons to our own data. +- Not the source of current state. Loro is. +- Not a Git-style parent chain. `previousCommit` is optional audit metadata. +- Not a product surface. The `/commits` class collection is not created. UI reads author and dates from the resource's genesis certificate, not by fetching `did:ad:commit:…`. +- Not required to stay on disk after apply, except genesis and rights / parent / destroy. -This approach has a couple of issues: +## How a write lands -- It costs a lot of resources, both for us and for the data suppliers. -- It's not real-time - we can only run this once every 24 ours (because of how costly it is). -- It's very prone to errors. We've had issues during all phases of Extraction, Transformation and Loading (ETL) processing. -- It causes privacy issues. When some data at the source is removed (because it contained faulty or privacy sensitive data), how do we learn about that? +1. The client edits the resource's Loro document. +2. It exports a compact binary delta (`loroUpdate`). +3. It signs a commit containing `subject`, `signer`, `createdAt`, `loroUpdate`, and a signature. +4. The server verifies the signature and the signer's rights, imports the Loro bytes, and materializes properties. Ordinary content commits are not stored as resources; genesis and rights/parent/destroy are. -Persisting and sharing state changes could solve these issues. -In order for this to work, we need to standardize this for all data suppliers. -We need a specification that is easy to understand for most developers. +HTTP `POST /commit` remains the fallback. The WebSocket `COMMIT` frame is the live path. -Keeping track of where data comes from is essential to knowing whether you can trust it - whether you consider it to be true. -When you want to persist data, that quickly becomes bothersome. -Atomic Data and Atomic Commits aim to make this easier by using cryptography for ensuring data comes from some particular source, and is therefore trustworthy. +## Motivation -If you want to know how Atomic Commits differ from other specs, see the [compare section](compare.md) +Systems that only publish *current state* make synchronization expensive: you +re-fetch everything and diff. Atomic Commits let a writer prove a specific +mutation. The mutation itself is a Loro delta, so two writers do not need a +lock or a linear history. Versioning, undo, and audit of *content* come from +the CRDT oplog. The signed envelope is what makes that mutation admissible. diff --git a/docs/src/interoperability/graph-database.md b/docs/src/interoperability/graph-database.md index bfc66cb16..0a040b765 100644 --- a/docs/src/interoperability/graph-database.md +++ b/docs/src/interoperability/graph-database.md @@ -15,8 +15,8 @@ After that, we'll explore how Atomic Data relates to some graph technologies. - **Authorization built-in**. Managing rights in a hierarchy (similar to how tools like Google Drive or filesystems work) enable you to have a high degree of control over read / write rights. - **Built-in easy to use GUI**. Managing content on Atomic-Server can be done by anyone, as its GUI is extremely easy to use and has a ton of features. - **Dynamic indexing**. Indexes are created by performing Queries, resulting in great performance - without needing to manually configure indexing. -- **Synchronization over WebSockets**. All changes (called [Commits](../commits/intro.md)) can be synchronized over WebSockets, allowing you to build realtime collaborative tools. -- **Event-sourced**. All changes are stored and reversible, giving you a full versioned history. +- **Synchronization over WebSockets**. Signed writes (called [Commits](../commits/intro.md)) can be synchronized over WebSockets, allowing you to build realtime collaborative tools. +- **CRDT versioning**. History comes from the Loro oplog; commits authorize writes rather than acting as an event log. - **Open source**. All code is MIT-licensed. ## Comparing Atomic Data to Neo4j diff --git a/docs/src/interoperability/rdf.md b/docs/src/interoperability/rdf.md index 08638c451..738c07de0 100644 --- a/docs/src/interoperability/rdf.md +++ b/docs/src/interoperability/rdf.md @@ -17,7 +17,7 @@ However, it does differ in some fundamental ways. - Atomic only allows those who control a resource's `subject` URL endpoint to edit the data. This means that you can't add triples about something that you don't control. - Atomic has no separate `datatype` field, but it requires that `Properties` (the resources that are shown when you follow a `predicate` value) specify a datatype. However, it is allowed to serialize the datatype explicitly, of course. - Atomic has no separate `language` field. -- Atomic has a native Event (state changes) model ([Atomic Commits](../commits/intro.md)), which enables communication of state changes +- Atomic has a native signed-write model ([Atomic Commits](../commits/intro.md)), which authorizes Loro CRDT updates - Atomic has a native Schema model ([Atomic Schema](../schema/intro.md)), which helps developers to know what data types they can expect (string, integer, link, array) - Atomic does not support Named Graphs. These should not be needed, because all statements should be retrievable by fetching the Subject of a resource. However, it _is_ allowed to include other resources in a response. @@ -225,9 +225,9 @@ This is why Atomic Data introduces a `shortname` field in Properties, which forc RDF lacks a clear solution for dealing with [ordered data](https://ontola.io/blog/ordered-data-in-rdf/), resulting in confusion when developers have to create lists of content. Adding an Array data type as a base data type helps solve this. ([discussion](https://github.com/ontola/atomic-data/issues/4)) -### Adding a native state changes standard +### Adding a native signed-write standard -There is no integrated standard for communicating state changes. +There is no integrated RDF standard for authorizing writes. Although [linked-delta](https://github.com/ontola/linked-delta) and [rdf-delta](https://afs.github.io/rdf-delta/) do exist, they aren't referred to by the RDF spec. I think developers need guidance when learning a new system such as RDF, and that's why [Atomic Commits](../commits/intro.md) is included in this book. diff --git a/docs/src/interoperability/solid.md b/docs/src/interoperability/solid.md index 274cd9b69..f9bb55a9c 100644 --- a/docs/src/interoperability/solid.md +++ b/docs/src/interoperability/solid.md @@ -38,22 +38,20 @@ This means that all Atomic Properties will have to exist on a publicly accessibl You can think of Atomic Data more like a (dynamic) SQL database that offers guarantees about its content type, and a Solid Pod more like a document store that takes in all kinds of content. Most of the differences have to do with how Atomic Schema aims to make linked data easier to work with, but that is covered in the previous [RDF chapter](./rdf.md). -## Atomic Data standardizes state changes (event sourcing) +## Atomic Data standardizes signed writes With Solid, you change a Resource by sending a POST request to the URL that you want to change. With Atomic, you change a Resource by sending a signed Commit that contains the requested changes to a Server. -Event sourcing means that all changes are stored (persisted) and used to calculate the current state of things. -In practice, this means that users get a couple of nice features for free: +A Commit is a signed envelope wrapping a Loro CRDT update. It authorizes the write; it is not an event-sourced history. Current state and version history live in the Resource's Loro document. -- **Versioning for all items by default**. Storing events means that these events can be _replayed_, which means you get to traverse time / undo / redo. -- **Edit / audit log for everything**. Events contain information about who made which change at which point in time. Can be useful for finding out why things are the way they are. -- **Easier to add query options / indexes**. Any system can play-back the events, which means that the events can be used as an API to add new query options / fill new indexes. This is especially useful if you want to add things like full-text search, or some geolocation index. +In practice, this means: -It also means that, compared to Solid, there is a relatively simple and strict API for changing data. -Atomic Data has a **uniform write API**. -All changes to data are done by posting Commits to the `/commits` endpoint of a Server. -This removes the need to think about differences between all sorts of HTTP methods like POST / PUT / PATCH, and how servers should reply to that. +- **Versioning for all items by default**. History is the CRDT oplog, not a replay of stored commit resources. +- **Attributable writes**. Every applied write is signed by an Agent. +- **Uniform write API**. All changes go through `POST /commit` (or the WebSocket `COMMIT` frame). This removes the need to think about differences between HTTP methods like POST / PUT / PATCH. + +All changes to data are done by posting Commits to the `/commit` endpoint of a Server. _EDIT: as of december 2021, Solid has introduced `.n3 patch` for standardizing state changes. Although this adds a uniform way of describing changes, it still lacks the power of Atomic Commits. It does not specify signatures, mention versioning, or deals with persisting changesets. On top of that, it is quite difficult to read or parse, being `.n3`._ @@ -130,7 +128,7 @@ I believe that as of today (february 2022), Atomic-Server has quite a few advant - **Lightweight** (8MB download, no runtime dependencies) - **HTTPS + HTTP2 support** with Built-in LetsEncrypt handshake. - **Browser GUI included** powered by [atomic-data-browser](https://github.com/atomicdata-dev/atomic-data-browser). Features dynamic forms, tables, authentication, theming and more. Easy to use! -- **Event-sourced versioning** / history powered by [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) +- **Versioning** / history from the Loro oplog, with writes authorized by [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) - **Many serialization options**: to JSON, [JSON-AD](https://docs.atomicdata.dev/core/serialization.html#json-ad), and various Linked Data / RDF formats (RDF/XML, N-Triples / Turtle / JSON-LD). - **Full-text search** with fuzzy search and various operators, often <3ms responses. - **Pagination, sorting and filtering** using [Atomic Collections](https://docs.atomicdata.dev/schema/collections.html) diff --git a/docs/src/interoperability/sql.md b/docs/src/interoperability/sql.md index dc582d15f..b5f15d39a 100644 --- a/docs/src/interoperability/sql.md +++ b/docs/src/interoperability/sql.md @@ -6,7 +6,7 @@ Atomic Data has some characteristics that make it similar and different from SQL - Atomic Data has a _dynamic_ schema. Any Resource could have different properties, so you can **add new properties** to your data without performing any migrations. However, the properties themselves are still validated (contrary to most NoSQL solutions) - Atomic Data uses **HTTP URLs** in its data, which means it's easy to **share and reuse**. - Atomic Data separates _reading_ and _writing_, whereas SQL has one language for both. -- Atomic Data has a standardized way of **storing changes** ([Commits](../commits/intro.md)) +- Atomic Data has a standardized way of **authorizing writes** ([Commits](../commits/intro.md)) ## Tables and Rows vs. Classes and Properties diff --git a/docs/src/roadmap.md b/docs/src/roadmap.md index 07cd0cd65..ba1323a84 100644 --- a/docs/src/roadmap.md +++ b/docs/src/roadmap.md @@ -22,7 +22,7 @@ This should help you understand how and where you may be able to contribute. - **[atomic-cli](https://crates.io/crates/atomic-cli) + [atomic-lib](https://docs.rs/atomic_lib/0.32.1/atomic_lib/)** (2020-07). The CLI functioned as the first platform to explore some of the most core ideas of Atomic Data, such as Properties and fetching. `atomic_lib` is the place where most logic resides. Written in Rust. - **[AtomicServer](https://github.com/atomicdata-dev/atomic-server/)** (2020-08). The server (using the same `atomic_lib` as the CLI) should be a fast, lightweight server that must be easy to set-up. Functions as a graph database with no dependencies. - **[Collections](schema/collections.md)** (2020-10). Allows users to perform basic queries, filtering, sorting and pagination. -- **[Commits](commits/intro.md)** (2020-11). Allow keeping track of an event-sourced log of all activities that mutate resources, which in turn allows for versioning and adding new types of indexes later on. +- **[Commits](commits/intro.md)** (2020-11). Signed write envelopes that authorize a Loro CRDT update. Originally conceived as an event-sourced log; current state and versioning now live in the resource's Loro document. - **[JSON-AD](core/json-ad.md)** (2021-02). Instead of the earlier proposed serialization format `.ad3`, we moved to the more familiar `json-ad`. - **[Atomic-Data-Browser](https://github.com/atomicdata-dev/atomic-data-browser)** (2021-02). We wanted typescript and react libraries, as well as a nice interactive GUI that works in the browser. It should implement all relevant parts of the specification. - **[Endpoints](endpoints.md)** (2021-03). Machine readable API endpoints (think Swagger / OpenAPI spec) for things like versioning, path traversal and more. diff --git a/docs/src/websockets.md b/docs/src/websockets.md index 77b268749..e7cb1d0ec 100644 --- a/docs/src/websockets.md +++ b/docs/src/websockets.md @@ -447,6 +447,8 @@ socket cannot know which commit produced it, and marks its next save as a genesis commit, which the responder rejects. A resource with no state answers `ERROR` `UNKNOWN` `No state`; an unreadable or missing subject answers `ERROR` `UNKNOWN` with the lookup error, on the same `request_id`. +That `lastCommit` id is a receipt for genesis detection, not a refetchable +resource. ## Persisted commits From 27da3e13aa48b909bccc93a8d91c24bc8abc33eb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 5 Sep 2026 10:33:59 +0000 Subject: [PATCH 4/8] docs(planning): verifiable History must survive clone Proofs have to replicate with the resource. Preferred path is a sibling Loro envelopes container so snapshot and SYNC_PUSH carry the signed log. Co-authored-by: joepmeindertsma --- planning/README.md | 3 +- planning/auditability-loro-history.md | 153 ++++++++++++++++++ ...commit-retention-and-state-certificates.md | 5 + 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 planning/auditability-loro-history.md diff --git a/planning/README.md b/planning/README.md index a0c7fbf9b..b961b2203 100644 --- a/planning/README.md +++ b/planning/README.md @@ -60,7 +60,7 @@ Remaining work, not "this file exists." | [`index-performance.md`](./index-performance.md) | First tranche shipped. Structural permission-check fix is `zones.md`, not built. | | [`disk-storage-and-persistence-optimization.md`](./disk-storage-and-persistence-optimization.md) | **Proposal.** Full-snapshot writes, no auto-compaction, O(file) open fsync. | | [`virtual-drive.md`](./virtual-drive.md) | **Shipped** as a local NFS mount in the Tauri desktop app (`desktop/src/vfs.rs`). Still proposal: headless-server mount, FUSE/WinFSP, native cloud-sync APIs, mobile providers. | -| [`commit-retention-and-state-certificates.md`](./commit-retention-and-state-certificates.md) | **Proposal** (DID wording predates the genesis-cert model, see its *Current* note). Commits stay signed write certificates; retention is node policy. | +| [`commit-retention-and-state-certificates.md`](./commit-retention-and-state-certificates.md) | **Proposal.** Commits stay signed write certificates; retention is node policy. Content-commit drop shipped; remaining is optional audit retention. | | [`p2p-presence.md`](./p2p-presence.md) | **Mostly built.** `EPHEMERAL 0x40` codec, peer send/receive and the server bridge are in (`lib/src/sync/iroh_e2e.rs` `e2e_presence_crosses_the_link_without_being_stored`). Remaining: two-device verification (M12), bandwidth measurement (OQ1). Scoped to your own devices by product choice. | | [`reticulum-sync.md`](./reticulum-sync.md) | **Proposal.** Atomic sync protocol over Reticulum. | | [`json-schema-code-first.md`](./json-schema-code-first.md) | **Proposal**; `defineSchema` + frozen `did:ad:` schemas in flight in PR #1262 (not on `develop`). Code-first JSON Schema → local DID-backed Class/Property resources. | @@ -90,6 +90,7 @@ Not top-level plans. Indexed so they do not go missing. | [`main-drive-and-paths.md`](./main-drive-and-paths.md) | Strategy. DID-branch deployment: root drive, legacy URLs, human-readable paths. | | [`actions.md`](./actions.md) | **Steps 1–4 shipped.** Registry drives ⌘M, ⌘K (capped prefix match), hotkeys, the shortcuts overlay/page, and simple AI tools. Remaining: MCP projection when a server exists. | | [`silent-failures.md`](./silent-failures.md) | Living log of error-handling failures that reported success (2026-08-21). Carries M8 from the pairing field test. | +| [`auditability-loro-history.md`](./auditability-loro-history.md) | Open. History = verifiable log for every replica, including new users (`git clone`). Envelopes live on the resource and catch-up must copy them; not a `/commits` class. | Closed decisions, as-built records, closed explorations and fixed notes live in [`completed/`](./completed/): the five decisions above, the 2026-07 sync diff --git a/planning/auditability-loro-history.md b/planning/auditability-loro-history.md new file mode 100644 index 000000000..563027446 --- /dev/null +++ b/planning/auditability-loro-history.md @@ -0,0 +1,153 @@ +# Verifiable History + +> **Status:** Open (2026-08-27). Companion to +> [`commit-retention-and-state-certificates.md`](./commit-retention-and-state-certificates.md) +> and [`authorization-sync.md`](./authorization-sync.md). +> +> Product goal: History is a list of **verifiable changes** — who, what, +> when, with cryptographic proof — for **every replica**, including a +> newly invited user. Same bar as `git clone` then `git log`. + +## Goal + +A new user who syncs a drive must be able to validate the log. Not only +the node that happened to see the live `COMMIT`. If they cannot, we have +not replaced what stored commits used to give. + +Each History row: + +| Shown | Source | Proof | +| --- | --- | --- | +| **What** | Loro checkout / diff | The oplog is the document | +| **When** | Envelope `createdAt` | Inside the signed JSON | +| **Who** | Envelope `signer` | Ed25519 over that JSON | +| **Proof** | Envelope `signature` | Same bytes `/commit` accepted | + +Today History is Loro-only (`Edited … by peer {hex}`). Content envelopes +are discarded after apply. A clone gets the document, not the log. + +## Git analogy + +| Git | Atomic | +| --- | --- | +| Blob / tree | Loro snapshot + oplog | +| Commit object (author, time, tree hash, signature if signed) | Signed envelope | +| `git clone` copies objects | Catch-up must copy envelopes **with** the snapshot | +| `git log` | History page | + +Snapshot-only `SYNC_PUSH` is `git clone --no-checkout` of the tree with +the `.git` directory empty. Unattributed History on a new device is that +failure mode. It is not an acceptable default. + +A linear `previousCommit` chain is **not** the git part we need (git +also has merge commits; Loro already merges). The git part we need is +**replicated commit objects**. + +## Split + +```text +Loro oplog = mergeable document (what) +Signed envelope = commit object (who, when, proof) +Both = replicate together +Graph / /commits = not a history store +``` + +Do not treat Loro change messages as “who.” They are plaintext. +Do not restore `/commits` as a queryable class. The log belongs to the +resource, like git objects belong to the repo — not to a site-wide +commit collection. + +## Requirement: proofs travel with the resource + +After apply, keep the signed envelope **on the resource’s replica +state**, so the next `SYNC_PUSH` / OPFS snapshot / Iroh catch-up +includes it. + +Preferred: a sibling Loro container on the same doc (e.g. `envelopes`: +commit-id → signed JSON-AD). Then today’s snapshot sync *is* clone of +the log. No `/commits` class, no extra Layer 2 trailer, no “blob table +the new user never sees.” + +Apply: + +1. Verify envelope, import `loroUpdate` into `properties` (as now). +2. Append the signed envelope to `envelopes` (CRDT map/list — concurrent + writers both land). +3. Stamp `lastCommit` for echo-dedup / genesis detection (as now). + +History: + +```text +Loro version → lastCommit / envelopes key → verify signature + → checkout → diff / restore +``` + +Missing envelope ⇒ **Unattributed** (legacy snapshot, truncated replica, +tamper). That is an error state, not the path for a new invitee. + +Authorization-critical commits (genesis / ACL / parent / destroy) stay +in the graph as the must-retain floor. Content envelopes live on the +resource. Neither is a site-wide event log. + +## Storage + +Same `Db` as the document (server redb, browser OPFS), because they are +part of that resource’s replica, not a server-only audit tape. + +If they live **in** the Loro doc, `Tree::LoroSnapshots` already holds +them. If they live in a side tree, bulk sync **must** send that tree +with the snapshot — same requirement, more wire. Prefer in-doc so +catch-up cannot forget them. + +Cost: the envelope repeats `loroUpdate`. That duplication is the +verifiable object, as a git commit repeats a pointer at content. A +header-only object (signer, createdAt, signature, hash of the change) +is a later size win; v1 stores the full signed body so verify matches +today’s `/commit` bytes. + +## Sync + +| Path | Must happen | +| --- | --- | +| Live `COMMIT` | Apply + persist envelope on the receiver (stop dropping it). | +| Bulk `SYNC_PUSH` | Snapshot includes envelopes (in-doc) **or** the push is incomplete. A new user who only got `properties` has an unverifiable log. | +| Offline local | OPFS snapshot includes envelopes; History verifies without network. | + +Layer 2 that imports a snapshot and ignores `envelopes` is a bug, not a +mode. See unsigned `SYNC_PUSH` in +[`authorization-sync.md`](./authorization-sync.md). + +## History row (target UI) + +- **Verified** — `{agent name} · {createdAt}` after Ed25519 check +- **Unattributed** — warning, not the normal row (legacy / incomplete replica) +- Diff and restore stay Loro checkouts +- No navigation to `did:ad:commit:…` as a document + +## What not to do + +- Local-only blob table that live writes keep and clones never get. +- “Unattributed is fine for new users.” +- Re-index envelopes as Atomic resources / `/commits`. +- Put the agent DID in the Loro change message and call it proof. +- Require a linear `previousCommit` chain. + +## Open questions + +1. **Container shape.** Loro map `envelopes[commitId] = json` vs list of + signed strings. Map is idempotent on retry (same id). +2. **Who writes the container.** Client includes it in `loroUpdate` at + sign time (envelope must then sign a doc that already contains itself + — chicken/egg) **or** server appends after verify (replica that only + has the client delta must apply the same append). Server-append after + verify is simpler; two replicas that both apply the same COMMIT must + append the same bytes so the CRDT converges. +3. **Concurrent edits.** Two envelopes, one merged document: both rows + verified, diffs from Loro. Correct, like two git commits on diverging + branches that later merge. +4. **Size / prune.** Full bodies grow the snapshot. Compaction that drops + old envelopes is a policy on that container, and it *removes* + verifiability for those versions — same as `git replace` / shallow + clone. Default is full log. +5. **Verify in WASM.** Same signature check as apply; fail closed to + Unattributed, never display a forged signer. diff --git a/planning/commit-retention-and-state-certificates.md b/planning/commit-retention-and-state-certificates.md index cb5208fb6..50ba57eab 100644 --- a/planning/commit-retention-and-state-certificates.md +++ b/planning/commit-retention-and-state-certificates.md @@ -12,6 +12,11 @@ > inherited down the parent chain), capped by node policy — see > "Per-resource retention" below. > +> **Shipped since:** ordinary content commits are discarded after apply; the +> must-retain floor (`AuthImpact::is_critical`) stays. History is Loro-only +> today. Target: a new replica can verify who changed what (`git clone` of +> the log) — [`auditability-loro-history.md`](./auditability-loro-history.md). +> > **Current (2026-06-10 server, 2026-07-10 browser).** DID identity is no > longer derived from the genesis *commit* signature. A `did:ad:` subject is > the signature of a small inline binary **genesis certificate** From 4ad03a49ee8c353493fa9f320f247a6c05bf3435 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 5 Sep 2026 10:45:21 +0000 Subject: [PATCH 5/8] style(lib): collapse store.ts Commit import to satisfy oxfmt CI failed on @tomic/lib format-check: the multi-line import of CommitBuilder / commitIdOf / Commit is a single-line import. Co-authored-by: joepmeindertsma --- browser/lib/src/store.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index 14c4bb384..ebfa0c302 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -6,11 +6,7 @@ import { setCookieAuthentication, } from './authentication.js'; import { Client, type FileOrFileLike } from './client.js'; -import { - CommitBuilder, - commitIdOf, - type Commit, -} from './commit.js'; +import { CommitBuilder, commitIdOf, type Commit } from './commit.js'; import { datatypeFromUrl, type Datatype } from './datatypes.js'; import { AtomicError, From 841a04e24f281f3dfa609104b1af088c764e3a24 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Sat, 5 Sep 2026 13:21:21 +0200 Subject: [PATCH 6/8] feat: keep signed envelopes per resource (Tree::Envelopes), attribute History Every applied signed commit leaves its JSON-AD on the resource it changed, keyed pure_id || createdAt || signature, written in the apply transaction. Not a resource, not indexed. Retention per node: latest (default, the envelope that produced the current state) or all (a signed audit log), via --envelope-retention / ATOMIC_ENVELOPE_RETENTION. Every commit's Loro change carries a token (browser drain token; Rust builder and create_did now tag too), so an envelope maps to the History version it produced. envelopes::attribute_history verifies signatures with the apply code, credits each token to one envelope (the genesis carrier only to a genesis envelope), and reports completeness. Read paths: GET /history-attribution (read-gated), WASM ClientDb.historyAttribution, Store.getHistoryAttribution merging both. History shows by Verified / Unverified / Unattributed. The destroy envelope on the tombstone value (#1370) is now the subject's latest row in this tree; the tombstone is a marker again. Tests: lib envelopes (8), server it history_attribution, browser lib history-attribution (5), e2e history assertions. Planning: decision doc amended (no #1274 gating, retention knob), auditability doc now Building. --- CHANGELOG.md | 12 + TESTING_COVERAGE.md | 3 + browser/CHANGELOG.md | 7 + .../src/routes/History/HistoryDesktopView.tsx | 3 +- .../src/routes/History/HistoryMobileView.tsx | 6 +- .../src/routes/History/HistoryRoute.tsx | 3 +- .../src/routes/History/HistoryViewProps.ts | 3 +- .../src/routes/History/VersionTitle.tsx | 68 +- .../src/routes/History/useVersions.ts | 39 +- browser/e2e/tests/e2e.spec.ts | 14 + browser/lib/src/client-db.node.ts | 14 + browser/lib/src/client-db.ts | 19 + browser/lib/src/client-db.worker.ts | 7 + browser/lib/src/history-attribution.test.ts | 129 ++++ browser/lib/src/history-attribution.ts | 146 +++++ browser/lib/src/index.ts | 6 + browser/lib/src/store.ts | 50 ++ docs/src/atomicserver/installation.md | 6 + docs/src/commits/versioning.md | 41 ++ lib/src/commit.rs | 12 + lib/src/db.rs | 40 +- lib/src/db/redb_store.rs | 3 + lib/src/db/sled_store.rs | 9 + lib/src/db/trees.rs | 9 + lib/src/envelopes.rs | 615 ++++++++++++++++++ lib/src/lib.rs | 2 + lib/src/resources.rs | 10 + lib/src/sync/tombstones.rs | 76 +-- planning/README.md | 6 +- planning/auditability-loro-history.md | 219 +++---- ...commit-retention-and-state-certificates.md | 11 +- .../commit-retention-floor-decision.md | 32 +- server/src/appstate.rs | 11 + server/src/config.rs | 6 + server/src/handlers/history_attribution.rs | 46 ++ server/src/handlers/mod.rs | 1 + server/src/routes.rs | 4 + server/tests/it/history_attribution.rs | 106 +++ server/tests/it/main.rs | 1 + wasm/src/lib.rs | 12 + 40 files changed, 1612 insertions(+), 195 deletions(-) create mode 100644 browser/lib/src/history-attribution.test.ts create mode 100644 browser/lib/src/history-attribution.ts create mode 100644 lib/src/envelopes.rs create mode 100644 server/src/handlers/history_attribution.rs create mode 100644 server/tests/it/history_attribution.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f09bdb0d1..bae38b5d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,18 @@ See [STATUS.md](server/STATUS.md) to learn more about which features will remain retained whether or not the client flagged `isGenesis`. Clients no longer chain `previousCommit`; apply no longer has a previous-commit validation gate. History no longer offers a "Show Commit" link at a discarded envelope. +- **Signed envelopes live on the resource (`Tree::Envelopes`).** Every + signed commit's JSON-AD is kept per resource, keyed by createdAt and + signature, in the same transaction as the state it signs. Not a + resource, not indexed. `--envelope-retention` / `ATOMIC_ENVELOPE_RETENTION` + is `latest` (the envelope that produced the current state; default) or + `all` (every envelope: a signed audit log). `GET /history-attribution?subject=` + (read-gated) answers who signed which Loro change, verified with the + apply code, plus whether every change is covered. Rust builder commits + and `create_did` now tag their Loro change like the browser does, so + History maps versions to signers. The destroy envelope on the tombstone + (added for `SYNC_DIFF.removeCommits`) is now the subject's latest row + in this tree. Envelopes do not yet travel in bulk sync or the vault. - **Missing-drive bootstrap is no longer a free pass (OQ5).** A `SYNC_PUSH` or live write for a drive this node has never stored goes diff --git a/TESTING_COVERAGE.md b/TESTING_COVERAGE.md index 6b6a7cd78..6fa4849b4 100644 --- a/TESTING_COVERAGE.md +++ b/TESTING_COVERAGE.md @@ -90,6 +90,9 @@ Two things worth knowing about the runners: | Engine-owned `SUB`/`UNSUB`: granted `SUB` is a session command, unreadable `SUB` answers `ERROR UNAUTHORIZED_READ` | `lib/src/sync/engine.rs` (`bootstrap_and_sub_tests`) | | Signed `SYNC_DIFF.removeCommits`: envelope applies regardless of connection agent, tampered envelope does not delete, envelope only handed to drive readers, replay after re-creation refused | `lib/src/sync/peer.rs` (`initiator_trust_tests`), `engine.rs` (`bootstrap_and_sub_tests`), `tombstones.rs`, `protocol.rs` | | `SyncSession` over an in-process `AtomicTransport` holds `AUTH` across frames | `lib/src/sync/session.rs` | +| Signed envelopes per resource: `latest`/`all` retention, time order, not indexed, verified attribution per Loro token, tampered envelope unverified, two writers, destroy fold | `lib/src/envelopes.rs` | +| `GET /history-attribution` names the verified signer and is read-gated | `server/tests/it/history_attribution.rs` | +| Attribution parse / version lookup / server+local merge | `browser/lib/src/history-attribution.test.ts` | | Engine-level two-store sync, private drives, blobs, live push | `lib/src/sync/tests.rs` | | RBSR reconciliation, drive hashing | `lib/src/sync/rbsr.rs`, `tests.rs` | | RBSR finds a remote-only subject sorting below every local one | `lib/src/sync/rbsr.rs` **and** `browser/lib/src/rbsr.test.ts` (regression, see below) | diff --git a/browser/CHANGELOG.md b/browser/CHANGELOG.md index b584ae62f..c6bcf2ea8 100644 --- a/browser/CHANGELOG.md +++ b/browser/CHANGELOG.md @@ -8,6 +8,13 @@ This changelog covers all five packages, as they are (for now) updated as a whol resources; author and dates come from the resource. History no longer navigates to a commit DID, and the Sync page log no longer links commit ids. Sequential saves no longer set `previousCommit`. +- History shows who signed each version. `Store.getHistoryAttribution(subject)` + merges the server's `GET /history-attribution` with the local ClientDb + (`historyAttribution`), and `VersionTitle` labels a version + `by Verified` (signature checked), `Unverified` (envelope present, + signature failed) or `by peer … Unattributed` (no envelope covers it). + `attributionForVersion`, `mergeHistoryAttributions`, `parseHistoryAttribution` + and the `HistoryAttribution` / `Attribution` types are exported from `@tomic/lib`. - [#1232](https://github.com/ontola/atomic-server/issues/1232) Unified actions: the ⌘K palette shows a capped Actions section for the current resource, hotkeys and the shortcuts overlay/page render from the action diff --git a/browser/data-browser/src/routes/History/HistoryDesktopView.tsx b/browser/data-browser/src/routes/History/HistoryDesktopView.tsx index 98becf576..88a97bf33 100644 --- a/browser/data-browser/src/routes/History/HistoryDesktopView.tsx +++ b/browser/data-browser/src/routes/History/HistoryDesktopView.tsx @@ -26,6 +26,7 @@ export function HistoryDesktopView({ onPreviousVersion, onSelectVersion, onVersionAccept, + attribution, }: HistoryViewProps) { const store = useStore(); @@ -67,7 +68,7 @@ export function HistoryDesktopView({ <> - <VersionTitle version={selectedVersion} /> + <VersionTitle version={selectedVersion} attribution={attribution} /> <StyledCard> <Tabs tabs={tabs} label='History'> <Card.Content> diff --git a/browser/data-browser/src/routes/History/HistoryMobileView.tsx b/browser/data-browser/src/routes/History/HistoryMobileView.tsx index dda9ee67a..d1370d67b 100644 --- a/browser/data-browser/src/routes/History/HistoryMobileView.tsx +++ b/browser/data-browser/src/routes/History/HistoryMobileView.tsx @@ -31,6 +31,7 @@ export function HistoryMobileView({ isCurrentVersion, onSelectVersion, onVersionAccept, + attribution, }: HistoryViewProps) { const [dialogProps, showDialog, closeDialog] = useDialog(); const store = useStore(); @@ -90,7 +91,10 @@ export function HistoryMobileView({ <Column fullHeight> {selectedVersion && ( <> - <VersionTitle version={selectedVersion} /> + <VersionTitle + version={selectedVersion} + attribution={attribution} + /> <StyledCard> <Tabs tabs={tabs} label='History'> <Card.Content> diff --git a/browser/data-browser/src/routes/History/HistoryRoute.tsx b/browser/data-browser/src/routes/History/HistoryRoute.tsx index e4ad0d2d1..6b1f144ea 100644 --- a/browser/data-browser/src/routes/History/HistoryRoute.tsx +++ b/browser/data-browser/src/routes/History/HistoryRoute.tsx @@ -31,7 +31,7 @@ function History(): JSX.Element { const isSmallScreen = useMediaQuery('(max-width: 500px)'); const [subject] = useCurrentSubject(); const resource = useResource(subject); - const { versions, loading, error } = useVersions(resource); + const { versions, attribution, loading, error } = useVersions(resource); const [selectedVersion, setSelectedVersion] = useState<Version | undefined>(); const resolvedVersion = @@ -132,6 +132,7 @@ function History(): JSX.Element { resource={resource} groupedVersions={groupedVersions} selectedVersion={selectedForView} + attribution={attribution} olderVersion={olderVersion} isCurrentVersion={isCurrentVersion} onNextVersion={nextVersion} diff --git a/browser/data-browser/src/routes/History/HistoryViewProps.ts b/browser/data-browser/src/routes/History/HistoryViewProps.ts index abcfee521..b77bd4f4e 100644 --- a/browser/data-browser/src/routes/History/HistoryViewProps.ts +++ b/browser/data-browser/src/routes/History/HistoryViewProps.ts @@ -1,4 +1,4 @@ -import { Resource, Version } from '@tomic/react'; +import { Resource, Version, type HistoryAttribution } from '@tomic/react'; export type GroupedVersions = { [key: string]: Version[]; @@ -8,6 +8,7 @@ export interface HistoryViewProps { resource: Resource; groupedVersions: GroupedVersions; selectedVersion: Version; + attribution: HistoryAttribution | null; olderVersion: Version | undefined; isCurrentVersion: boolean; onNextVersion: () => void; diff --git a/browser/data-browser/src/routes/History/VersionTitle.tsx b/browser/data-browser/src/routes/History/VersionTitle.tsx index 607f5504c..87f28a79e 100644 --- a/browser/data-browser/src/routes/History/VersionTitle.tsx +++ b/browser/data-browser/src/routes/History/VersionTitle.tsx @@ -1,6 +1,12 @@ -import type { Version } from '@tomic/react'; +import { + attributionForVersion, + type HistoryAttribution, + type Version, +} from '@tomic/react'; +import { styled } from 'styled-components'; import type { JSX } from 'react'; +import { ResourceInline } from '../../views/ResourceInline/ResourceInline'; const formatter = new Intl.DateTimeFormat('default', { month: 'long', @@ -13,16 +19,70 @@ const formatter = new Intl.DateTimeFormat('default', { export interface VersionTitleProps { version: Version; + /** Signed-envelope attribution for the resource, when any is retained. */ + attribution?: HistoryAttribution | null; } -export function VersionTitle({ version }: VersionTitleProps): JSX.Element { + +/** + * "Edited <when> by <whom>". The signer comes from a signed envelope whose + * Loro change token matches this version, and is labelled Verified when the + * answering node checked the signature. A version no envelope claims falls + * back to the Loro peer id: that is who typed, not a proof of who signed. + */ +export function VersionTitle({ + version, + attribution, +}: VersionTitleProps): JSX.Element { const date = new Date(version.timestamp); const formattedDate = formatter.format(date); + const signed = attributionForVersion(version, attribution); return ( <span> Edited <time dateTime={date.toISOString()}>{formattedDate}</time> - {version.peer && <> by peer {version.peer.slice(0, 8)}...</>} - {version.message && <> — {version.message}</>} + {signed ? ( + <> + {' by '} + <ResourceInline subject={signed.signer} />{' '} + <Badge + $verified={signed.verified} + title={ + signed.verified + ? "Signature checked against the signer's key" + : 'Envelope present, but its signature did not verify' + } + data-testid='version-attribution' + > + {signed.verified ? 'Verified' : 'Unverified'} + </Badge> + </> + ) : ( + version.peer && ( + <> + {' by peer '} + {version.peer.slice(0, 8)}...{' '} + <Badge + $verified={false} + title='No signed envelope covers this change on this node' + data-testid='version-attribution' + > + Unattributed + </Badge> + </> + ) + )} + {version.message && !signed && <> — {version.message}</>} </span> ); } + +const Badge = styled.span<{ $verified: boolean }>` + display: inline-block; + padding: 0 0.4em; + border-radius: ${p => p.theme.radius}; + font-size: 0.8em; + line-height: 1.6; + color: ${p => (p.$verified ? 'white' : p.theme.colors.textLight)}; + background-color: ${p => + p.$verified ? p.theme.colors.main : p.theme.colors.bg1}; +`; diff --git a/browser/data-browser/src/routes/History/useVersions.ts b/browser/data-browser/src/routes/History/useVersions.ts index 7dbe4141a..3b40eda18 100644 --- a/browser/data-browser/src/routes/History/useVersions.ts +++ b/browser/data-browser/src/routes/History/useVersions.ts @@ -1,8 +1,19 @@ -import { Resource, Version, unknownSubject } from '@tomic/react'; +import { + Resource, + Version, + unknownSubject, + useStore, + type HistoryAttribution, +} from '@tomic/react'; import { useState, useEffect, useRef } from 'react'; export interface UseVersionsResult { versions: Version[]; + /** + * Who signed which version, from the signed envelopes the server and the + * local ClientDb kept. Null while loading or when nothing is retained. + */ + attribution: HistoryAttribution | null; loading: boolean; error: Error | undefined; } @@ -12,11 +23,35 @@ export interface UseVersionsResult { * Instant — no network requests needed, no progress bar. */ export function useVersions(resource: Resource): UseVersionsResult { + const store = useStore(); const [versions, setVersions] = useState<Version[]>([]); + const [attribution, setAttribution] = useState<HistoryAttribution | null>( + null, + ); const [loading, setLoading] = useState<boolean>(true); const [error, setError] = useState<Error | undefined>(undefined); const isRunning = useRef(false); + // Attribution is a network round-trip (and a WASM replay), so it lands + // after the versions do; the list renders unattributed until then. + useEffect(() => { + if (resource.subject === unknownSubject || resource.loading) { + return; + } + + let cancelled = false; + store + .getHistoryAttribution(resource.subject) + .then(report => { + if (!cancelled) setAttribution(report); + }) + .catch(() => undefined); + + return () => { + cancelled = true; + }; + }, [store, resource.subject, resource.loading, versions.length]); + useEffect(() => { if (resource.subject === unknownSubject || resource.loading) { return; @@ -40,5 +75,5 @@ export function useVersions(resource: Resource): UseVersionsResult { } }, [resource, resource.loading]); - return { versions, loading, error }; + return { versions, attribution, loading, error }; } diff --git a/browser/e2e/tests/e2e.spec.ts b/browser/e2e/tests/e2e.spec.ts index 8b25bee06..5636d4e30 100644 --- a/browser/e2e/tests/e2e.spec.ts +++ b/browser/e2e/tests/e2e.spec.ts @@ -817,12 +817,26 @@ test.describe('data-browser', async () => { page.getByRole('heading', { name: 'History of Second Title', level: 1 }), ).toBeVisible(); + // The current version is signed by this session's agent and the server + // kept its envelope (`Tree::Envelopes`, `latest` retention), so History + // attributes it as Verified once `/history-attribution` answers. + await expect(page.getByTestId('version-attribution')).toHaveText( + 'Verified', + { timeout: 15_000 }, + ); + await selectHistoryVersionShowing(page, 'First Title'); await expect( page.getByText('First Title', { exact: true }).first(), ).toBeVisible(); + // Under `latest` retention only the newest envelope is kept, so the + // older version has no proof and must say so rather than guess. + await expect(page.getByTestId('version-attribution')).toHaveText( + 'Unattributed', + ); + // Enabled only once the selected version differs from the current one, so // wait for that rather than racing the selection above. const restore = page.getByRole('button', { name: 'Restore this version' }); diff --git a/browser/lib/src/client-db.node.ts b/browser/lib/src/client-db.node.ts index 38de9dbd6..dc80067b3 100644 --- a/browser/lib/src/client-db.node.ts +++ b/browser/lib/src/client-db.node.ts @@ -7,6 +7,10 @@ * use, keep `ClientDbWorker`. */ +import { + parseHistoryAttribution, + type HistoryAttribution, +} from './history-attribution.js'; import { readFile } from 'node:fs/promises'; import type { ClientDbQueryOpts, ClientDbQueryResult } from './client-db.js'; @@ -237,6 +241,16 @@ export class NodeClientDb { return (r as Uint8Array | null) ?? null; } + async historyAttribution( + subject: string, + ): Promise<HistoryAttribution | null> { + const db = this.requireDb(); + + if (typeof db.historyAttribution !== 'function') return null; + + return parseHistoryAttribution(await db.historyAttribution(subject)); + } + async putBlob(hash: Uint8Array, data: Uint8Array): Promise<void> { this.requireDb().putBlob(hash, data); } diff --git a/browser/lib/src/client-db.ts b/browser/lib/src/client-db.ts index 67b7546a6..900a54eb7 100644 --- a/browser/lib/src/client-db.ts +++ b/browser/lib/src/client-db.ts @@ -25,6 +25,10 @@ * ``` */ +import { + parseHistoryAttribution, + type HistoryAttribution, +} from './history-attribution.js'; import type { Aggregation, AggregateOutcome, @@ -726,6 +730,21 @@ export class ClientDbWorker { return (r as Uint8Array | null) ?? null; } + /** Who signed `subject`'s history, from the envelopes this client applied + * itself (`atomic_lib::envelopes`). Null when the WASM build predates the + * accessor or the worker is unavailable. */ + async historyAttribution( + subject: string, + ): Promise<HistoryAttribution | null> { + try { + const r = await this.send({ type: 'historyAttribution', subject }); + + return parseHistoryAttribution(r); + } catch { + return null; + } + } + async putBlob(hash: Uint8Array, data: Uint8Array): Promise<void> { await this.send({ type: 'putBlob', hash, data }); } diff --git a/browser/lib/src/client-db.worker.ts b/browser/lib/src/client-db.worker.ts index 72df3dbd1..a42609bbe 100644 --- a/browser/lib/src/client-db.worker.ts +++ b/browser/lib/src/client-db.worker.ts @@ -75,6 +75,7 @@ export type WorkerRequest = | { id: number; type: 'exportAllResources' } | { id: number; type: 'importAllResources'; jsonArray: string } | { id: number; type: 'getLoroSnapshot'; subject: string } + | { id: number; type: 'historyAttribution'; subject: string } | { id: number; type: 'putBlob'; hash: Uint8Array; data: Uint8Array } | { id: number; type: 'getBlob'; hash: Uint8Array } | { id: number; type: 'blake3Hash'; data: Uint8Array } @@ -304,6 +305,12 @@ async function handleMessage(msg: WorkerRequest): Promise<unknown> { return db!.getLoroSnapshot(msg.subject); } + case 'historyAttribution': { + await ensureInit(); + + return (await db!.historyAttribution(msg.subject)) as string; + } + case 'putBlob': { await ensureInit(); db!.putBlob(msg.hash, msg.data); diff --git a/browser/lib/src/history-attribution.test.ts b/browser/lib/src/history-attribution.test.ts new file mode 100644 index 000000000..bb2bc63e5 --- /dev/null +++ b/browser/lib/src/history-attribution.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from 'vitest'; +import { + attributionForVersion, + mergeHistoryAttributions, + parseHistoryAttribution, + type Attribution, + type HistoryAttribution, +} from './history-attribution.js'; + +const alice = 'did:ad:agent:alice'; +const bob = 'did:ad:agent:bob'; + +function attribution(overrides: Partial<Attribution>): Attribution { + return { + signer: alice, + createdAt: 1, + signature: 'sig', + verified: true, + tokens: [], + destroy: false, + genesis: false, + ...overrides, + }; +} + +describe('parseHistoryAttribution', () => { + it('reads the server JSON (snake_case createdAt) and the WASM string', () => { + const wire = { + subject: 'did:ad:x', + retention: 'all', + complete: true, + attributions: [ + { + signer: alice, + created_at: 42, + signature: 'a', + verified: true, + tokens: [alice], + destroy: false, + genesis: true, + }, + { signer: bob, created_at: 43, signature: 'b', verified: false, tokens: ['c-1'] }, + ], + }; + const fromObject = parseHistoryAttribution(wire); + const fromString = parseHistoryAttribution(JSON.stringify(wire)); + + expect(fromObject).toEqual(fromString); + expect(fromObject?.attributions).toHaveLength(2); + expect(fromObject?.attributions[0]).toMatchObject({ + signer: alice, + createdAt: 42, + genesis: true, + verified: true, + }); + expect(fromObject?.attributions[1]).toMatchObject({ + signer: bob, + verified: false, + destroy: false, + genesis: false, + }); + expect(fromObject?.complete).toBe(true); + }); + + it('returns null for malformed input and skips malformed rows', () => { + expect(parseHistoryAttribution('not json')).toBeNull(); + expect(parseHistoryAttribution({ nope: true })).toBeNull(); + const report = parseHistoryAttribution({ + attributions: [{ signer: alice }, 'junk', null], + }); + expect(report?.attributions).toEqual([]); + expect(report?.complete).toBe(false); + }); +}); + +describe('attributionForVersion', () => { + const report: HistoryAttribution = { + subject: 'did:ad:x', + retention: 'all', + complete: true, + attributions: [ + attribution({ signature: 'g', tokens: [alice], genesis: true }), + attribution({ signer: bob, signature: 'e', tokens: ['c-7'] }), + ], + }; + + it('maps a version to the envelope carrying its token', () => { + expect(attributionForVersion({ message: 'c-7' }, report)?.signer).toBe(bob); + expect(attributionForVersion({ message: alice }, report)?.genesis).toBe( + true, + ); + }); + + it('leaves untokened or unclaimed versions unattributed', () => { + expect(attributionForVersion({ message: undefined }, report)).toBeUndefined(); + expect(attributionForVersion({ message: 'c-9' }, report)).toBeUndefined(); + expect(attributionForVersion({ message: 'c-7' }, null)).toBeUndefined(); + }); +}); + +describe('mergeHistoryAttributions', () => { + it('unions by signature, prefers verified, sorts by time', () => { + const server: HistoryAttribution = { + subject: 'did:ad:x', + retention: 'latest', + complete: false, + attributions: [ + attribution({ signature: 'b', createdAt: 2, verified: false }), + ], + }; + const local: HistoryAttribution = { + subject: 'did:ad:x', + retention: 'all', + complete: true, + attributions: [ + attribution({ signature: 'a', createdAt: 1 }), + attribution({ signature: 'b', createdAt: 2, verified: true }), + ], + }; + const merged = mergeHistoryAttributions(server, local); + + expect(merged?.attributions.map(a => a.signature)).toEqual(['a', 'b']); + expect(merged?.attributions[1].verified).toBe(true); + expect(merged?.retention).toBe('all'); + expect(merged?.complete).toBe(true); + expect(mergeHistoryAttributions(null, local)).toBe(local); + expect(mergeHistoryAttributions(null, null)).toBeNull(); + }); +}); diff --git a/browser/lib/src/history-attribution.ts b/browser/lib/src/history-attribution.ts new file mode 100644 index 000000000..726567f16 --- /dev/null +++ b/browser/lib/src/history-attribution.ts @@ -0,0 +1,146 @@ +/** + * Who signed a resource's history. + * + * A node keeps signed commit envelopes per resource (`atomic_lib::envelopes`, + * `Tree::Envelopes`): the envelope that produced the current state (`latest` + * retention) or every envelope (`all`). Verifying them is replay: the + * signature is checked with the same code apply uses, and the envelope's Loro + * update names the change tokens it introduced. History buckets versions by + * those tokens, so a version maps to its signer by lookup. Anything not + * covered is unattributed, never a guessed signer. + * + * The report comes from `GET /history-attribution?subject=` on the connected + * server, or from the local ClientDb for resources it applied itself. + */ + +import type { Version } from './resource.js'; + +export interface Attribution { + /** Agent subject that signed the commit. */ + signer: string; + /** Commit `createdAt`, Unix milliseconds. */ + createdAt: number; + signature: string; + /** The signature checked out against the signer's key on the answering node. */ + verified: boolean; + /** Loro change messages (drain tokens) this envelope introduced. */ + tokens: string[]; + destroy: boolean; + genesis: boolean; +} + +export interface HistoryAttribution { + /** Pure id of the resource. */ + subject: string; + /** `latest` or `all`: the answering node's envelope retention. */ + retention: 'latest' | 'all' | string; + /** Oldest first. */ + attributions: Attribution[]; + /** + * Every client-authored change in the oplog is claimed by a verified + * envelope. `false` means History has versions nobody can be held to. + */ + complete: boolean; +} + +/** Parse the server / WASM JSON. Returns null for anything malformed. */ +export function parseHistoryAttribution( + input: unknown, +): HistoryAttribution | null { + const data = + typeof input === 'string' + ? (() => { + try { + return JSON.parse(input); + } catch { + return undefined; + } + })() + : input; + + if (!data || typeof data !== 'object') return null; + const record = data as Record<string, unknown>; + + if (!Array.isArray(record.attributions)) return null; + + const attributions: Attribution[] = []; + + for (const raw of record.attributions) { + if (!raw || typeof raw !== 'object') continue; + const a = raw as Record<string, unknown>; + + if (typeof a.signer !== 'string' || typeof a.signature !== 'string') { + continue; + } + + attributions.push({ + signer: a.signer, + createdAt: Number(a.created_at ?? a.createdAt ?? 0), + signature: a.signature, + verified: a.verified === true, + tokens: Array.isArray(a.tokens) + ? a.tokens.filter((t): t is string => typeof t === 'string') + : [], + destroy: a.destroy === true, + genesis: a.genesis === true, + }); + } + + return { + subject: typeof record.subject === 'string' ? record.subject : '', + retention: typeof record.retention === 'string' ? record.retention : '', + attributions, + complete: record.complete === true, + }; +} + +/** + * The attribution that signed this version, by its change token. A version + * without a token (server bookkeeping) or whose token no retained envelope + * carries is unattributed. + */ +export function attributionForVersion( + version: Pick<Version, 'message'>, + report: HistoryAttribution | null | undefined, +): Attribution | undefined { + if (!report || !version.message) return undefined; + const token = version.message; + + return report.attributions.find(a => a.tokens.includes(token)); +} + +/** + * Union of two reports about the same resource (server and local ClientDb), + * de-duplicated by signature, oldest first. A verified attribution wins over + * an unverified one for the same signature. `complete` holds only when the + * merged set covers everything either side saw. + */ +export function mergeHistoryAttributions( + a: HistoryAttribution | null | undefined, + b: HistoryAttribution | null | undefined, +): HistoryAttribution | null { + if (!a) return b ?? null; + if (!b) return a; + + const bySignature = new Map<string, Attribution>(); + + for (const attribution of [...a.attributions, ...b.attributions]) { + const existing = bySignature.get(attribution.signature); + + if (!existing || (!existing.verified && attribution.verified)) { + bySignature.set(attribution.signature, attribution); + } + } + + const attributions = [...bySignature.values()].sort( + (x, y) => x.createdAt - y.createdAt, + ); + + return { + subject: a.subject || b.subject, + retention: + a.retention === 'all' || b.retention === 'all' ? 'all' : a.retention, + attributions, + complete: a.complete || b.complete, + }; +} diff --git a/browser/lib/src/index.ts b/browser/lib/src/index.ts index 2d69301f7..fd1e17112 100644 --- a/browser/lib/src/index.ts +++ b/browser/lib/src/index.ts @@ -70,6 +70,12 @@ export * from './loro-loader.js'; export * from './presence.js'; export * from './CryptoProvider.js'; export { ClientDbWorker } from './client-db.js'; +export { + attributionForVersion, + mergeHistoryAttributions, + parseHistoryAttribution, +} from './history-attribution.js'; +export type { Attribution, HistoryAttribution } from './history-attribution.js'; export type { ClientDbQueryOpts, ClientDbQueryResult } from './client-db.js'; export { LocalOutbox, diff --git a/browser/lib/src/store.ts b/browser/lib/src/store.ts index ebfa0c302..f40ca6095 100644 --- a/browser/lib/src/store.ts +++ b/browser/lib/src/store.ts @@ -1,9 +1,15 @@ +import { + mergeHistoryAttributions, + parseHistoryAttribution, + type HistoryAttribution, +} from './history-attribution.js'; import { ulid } from 'ulidx'; import type { Agent } from './agent.js'; import { canonicalDriveHash } from './canonical-drive-hash.js'; import { removeCookieAuthentication, setCookieAuthentication, + signRequest, } from './authentication.js'; import { Client, type FileOrFileLike } from './client.js'; import { CommitBuilder, commitIdOf, type Commit } from './commit.js'; @@ -3394,6 +3400,50 @@ export class Store { return this.serverUrl; } + /** + * Who signed `subject`'s history: the verified signer per Loro change + * token, from the signed envelopes the connected server kept + * (`GET /history-attribution`, read-gated) merged with those this client + * applied itself (ClientDb). Null when neither has anything. Never + * guesses: a version whose token no envelope carries stays unattributed. + */ + public async getHistoryAttribution( + subject: string, + ): Promise<HistoryAttribution | null> { + const [remote, local] = await Promise.all([ + this.fetchHistoryAttributionFromServer(subject), + this.getClientDb()?.historyAttribution(subject) ?? Promise.resolve(null), + ]); + + return mergeHistoryAttributions(remote, local); + } + + private async fetchHistoryAttributionFromServer( + subject: string, + ): Promise<HistoryAttribution | null> { + if (!this.serverUrl) return null; + + try { + const url = new URL('/history-attribution', this.serverUrl); + url.searchParams.set('subject', subject); + const agent = this.getAgent(); + // Sign the URL being fetched: the server rebuilds the signed message + // from the request it received, query string included. + const headers = agent + ? await signRequest(url.toString(), agent, { + Accept: 'application/json', + }) + : { Accept: 'application/json' }; + const res = await fetch(url.toString(), { headers }); + + if (!res.ok) return null; + + return parseHistoryAttribution(await res.json()); + } catch { + return null; + } + } + /** * Returns the Currently set Agent, returns null if there is none. Make sure * to first run `store.setAgent()`. diff --git a/docs/src/atomicserver/installation.md b/docs/src/atomicserver/installation.md index a74bde006..dcdbf6892 100644 --- a/docs/src/atomicserver/installation.md +++ b/docs/src/atomicserver/installation.md @@ -380,6 +380,12 @@ Options: [env: ATOMIC_DEVELOPMENT=] + --envelope-retention <ENVELOPE_RETENTION> + Which signed commit envelopes this node keeps per resource: `latest` (the envelope that produced the current state; the default) or `all` (every envelope, so History shows a verified signer per change) + + [env: ATOMIC_ENVELOPE_RETENTION=] + [default: latest] + --domain <DOMAIN> The origin domain where the app is hosted, without the port and schema values diff --git a/docs/src/commits/versioning.md b/docs/src/commits/versioning.md index 0b02ca6b8..961bc4c7c 100644 --- a/docs/src/commits/versioning.md +++ b/docs/src/commits/versioning.md @@ -24,3 +24,44 @@ A static resource has a _content addressable_ URL, which means that its URL will - Serialize all Atoms of the Subject (the entire Resource) as Atomic-NDJSON - Sort all lines (every atom) alphabetically + + +## Who signed a version + +Every applied commit leaves its signed JSON-AD on the resource it changed, in +a side tree on the node (`Tree::Envelopes`). It is not a resource and it is +not indexed: it never appears in queries or collections. A node keeps either +the envelope that produced the current state (`--envelope-retention latest`, +the default) or every envelope (`all`), which turns the Loro history into a +signed audit log. + +Each commit's Loro change carries a token in its message, and the envelope +that introduced that change carries the same token inside its `loroUpdate`. +That is how a version in History maps to its signer. + +`GET /history-attribution?subject=<subject>` returns, for a resource the +caller may read: + +```json +{ + "subject": "did:ad:…", + "retention": "all", + "complete": true, + "attributions": [ + { + "signer": "did:ad:agent:…", + "created_at": 1757060000000, + "signature": "…", + "verified": true, + "tokens": ["c-1a07140ba9b-uvdzz0"], + "destroy": false, + "genesis": false + } + ] +} +``` + +`verified` means the answering node re-checked the signature with the same +code it applies commits with. `complete` means every client-authored change +in the oplog is claimed by a verified envelope. A version no envelope covers +is shown as *Unattributed*; a signer is never guessed. diff --git a/lib/src/commit.rs b/lib/src/commit.rs index 96bc63814..bc7f529f9 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -295,6 +295,11 @@ impl Commit { } } doc.set_property(urls::GENESIS, &crate::values::Value::String(cert_b64))?; + // The genesis change carries the creator's subject as its message, + // exactly as the browser writes it: `createdBy` reads it, and the + // signed genesis envelope is matched back to this change by it + // (`crate::envelopes::attribute_history`). + doc.commit_with_message(agent.subject.as_str()); let loro_update = Some(doc.export_snapshot()); let mut commit = Commit { @@ -1404,6 +1409,13 @@ async fn sign_at( for prop in &commitbuilder.remove { doc.remove_property(prop)?; } + // One tokened change per commit, like the browser: history buckets + // versions by it and the envelope is attributed to it. + doc.commit_with_message(&format!( + "c-{:x}-{}", + crate::utils::now(), + crate::utils::random_string(6) + )); Some(doc.export_snapshot()) } else { commitbuilder.loro_update diff --git a/lib/src/db.rs b/lib/src/db.rs index e45770590..90c773b75 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -328,6 +328,10 @@ pub struct Db { /// self-hosted / local-first nodes are unrestricted; a managed node /// installs a concrete policy via [`Db::set_sync_policy`]. sync_policy: Arc<RwLock<Arc<dyn crate::sync::policy::SyncPolicy>>>, + /// Which signed envelopes `apply_commit` keeps per resource + /// (`crate::envelopes`). `Latest` by default; a node that wants a signed + /// audit log runs `All`. + envelope_retention: Arc<RwLock<crate::envelopes::EnvelopeRetention>>, /// Short-lived hash → (drive-subject, requested-at) map for blob hashes /// the server has asked a peer for (via `BLOB_REQUEST`, emitted from /// `import_sync_push` for an already-admitted drive). Consulted when @@ -366,6 +370,22 @@ impl Db { } /// The currently-installed sync policy. + /// Set how many signed envelopes are kept per resource. Takes effect on + /// the next `apply_commit`; existing rows are pruned when their resource + /// is next written. + pub fn set_envelope_retention(&self, retention: crate::envelopes::EnvelopeRetention) { + if let Ok(mut guard) = self.envelope_retention.write() { + *guard = retention; + } + } + + pub fn envelope_retention(&self) -> crate::envelopes::EnvelopeRetention { + self.envelope_retention + .read() + .map(|g| *g) + .unwrap_or_default() + } + pub fn sync_policy(&self) -> Arc<dyn crate::sync::policy::SyncPolicy> { self.sync_policy .read() @@ -432,6 +452,7 @@ impl Db { subject_locks: Default::default(), base_domain, sync_policy: default_sync_policy(), + envelope_retention: Arc::new(RwLock::new(Default::default())), pending_blob_requests: Arc::new(RwLock::new(HashMap::new())), }; @@ -470,6 +491,7 @@ impl Db { subject_locks: Default::default(), base_domain, sync_policy: default_sync_policy(), + envelope_retention: Arc::new(RwLock::new(Default::default())), pending_blob_requests: Arc::new(RwLock::new(HashMap::new())), }; @@ -504,6 +526,7 @@ impl Db { subject_locks: Default::default(), base_domain, sync_policy: default_sync_policy(), + envelope_retention: Arc::new(RwLock::new(Default::default())), pending_blob_requests: Arc::new(RwLock::new(HashMap::new())), }; @@ -599,6 +622,7 @@ impl Db { subject_locks: Default::default(), base_domain, sync_policy: default_sync_policy(), + envelope_retention: Arc::new(RwLock::new(Default::default())), pending_blob_requests: Arc::new(RwLock::new(HashMap::new())), }; @@ -750,6 +774,7 @@ impl Db { subject_locks: Default::default(), base_domain, sync_policy: default_sync_policy(), + envelope_retention: Arc::new(RwLock::new(Default::default())), pending_blob_requests: Arc::new(RwLock::new(HashMap::new())), }; @@ -3148,6 +3173,12 @@ impl Storelike for Db { } } + // The signed envelope itself lives in `Tree::Envelopes`, keyed by the + // resource it is about, in the same transaction as the state it + // signs. This is what lets any node that holds the resource say who + // signed it, independent of the commit rows above. + crate::envelopes::record_ops(store, &commit_response, &mut transaction)?; + match (&commit_response.resource_old, &commit_response.resource_new) { (None, None) => { if !commit_response.commit.destroy.unwrap_or(false) { @@ -3166,13 +3197,10 @@ impl Storelike for Db { .destroy .expect("Resource was removed but `commit.destroy` was not set!")); let subject: Subject = commit_response.commit.subject.clone(); + // `remove_resource` records the tombstone; the signed destroy + // is the envelope row `record_ops` queued above, which is what + // `SYNC_DIFF.removeCommits` carries. self.remove_resource(&subject).await?; - // `remove_resource` records an unsigned tombstone. Overlay the - // signed destroy so bulk `SYNC_DIFF.removeCommits` can carry - // the same envelope the live `COMMIT` path forwards. - if let Ok(json) = commit_response.commit_resource.to_json_ad(None) { - crate::sync::tombstones::record_destroy_envelope(self, subject.as_str(), &json); - } } _ => {} }; diff --git a/lib/src/db/redb_store.rs b/lib/src/db/redb_store.rs index afd0bd04c..7f3a61011 100644 --- a/lib/src/db/redb_store.rs +++ b/lib/src/db/redb_store.rs @@ -38,6 +38,7 @@ const TABLE_SEARCH_DOC_TOKENS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("search_doc_tokens_v1"); const TABLE_SEARCH_TRIGRAMS: TableDefinition<&[u8], &[u8]> = TableDefinition::new("search_trigrams_v1"); +const TABLE_ENVELOPES: TableDefinition<&[u8], &[u8]> = TableDefinition::new("envelopes_v1"); fn table_def(tree: Tree) -> TableDefinition<'static, &'static [u8], &'static [u8]> { match tree { @@ -55,6 +56,7 @@ fn table_def(tree: Tree) -> TableDefinition<'static, &'static [u8], &'static [u8 Tree::SearchDocs => TABLE_SEARCH_DOCS, Tree::SearchDocTokens => TABLE_SEARCH_DOC_TOKENS, Tree::SearchTrigrams => TABLE_SEARCH_TRIGRAMS, + Tree::Envelopes => TABLE_ENVELOPES, } } @@ -74,6 +76,7 @@ fn create_all_tables(tx: &redb::WriteTransaction) { Tree::SearchDocs, Tree::SearchDocTokens, Tree::SearchTrigrams, + Tree::Envelopes, ] { let _ = tx.open_table(table_def(tree)); } diff --git a/lib/src/db/sled_store.rs b/lib/src/db/sled_store.rs index ad7d88ed5..ec95780b1 100644 --- a/lib/src/db/sled_store.rs +++ b/lib/src/db/sled_store.rs @@ -28,6 +28,7 @@ pub struct SledStore { search_docs: sled::Tree, search_doc_tokens: sled::Tree, search_trigrams: sled::Tree, + envelopes: sled::Tree, } impl SledStore { @@ -55,6 +56,7 @@ impl SledStore { let search_docs = db.open_tree(Tree::SearchDocs)?; let search_doc_tokens = db.open_tree(Tree::SearchDocTokens)?; let search_trigrams = db.open_tree(Tree::SearchTrigrams)?; + let envelopes = db.open_tree(Tree::Envelopes)?; Ok(SledStore { db, @@ -72,6 +74,7 @@ impl SledStore { search_docs, search_doc_tokens, search_trigrams, + envelopes, }) } @@ -96,6 +99,7 @@ impl SledStore { Tree::SearchDocs => &self.search_docs, Tree::SearchDocTokens => &self.search_doc_tokens, Tree::SearchTrigrams => &self.search_trigrams, + Tree::Envelopes => &self.envelopes, } } } @@ -179,6 +183,7 @@ impl KvStore for SledStore { let mut batch_search_docs = sled::Batch::default(); let mut batch_search_doc_tokens = sled::Batch::default(); let mut batch_search_trigrams = sled::Batch::default(); + let mut batch_envelopes = sled::Batch::default(); for op in operations { let batch = match op.tree { @@ -196,6 +201,7 @@ impl KvStore for SledStore { Tree::SearchDocs => &mut batch_search_docs, Tree::SearchDocTokens => &mut batch_search_doc_tokens, Tree::SearchTrigrams => &mut batch_search_trigrams, + Tree::Envelopes => &mut batch_envelopes, }; match op.method { Method::Insert => { @@ -263,6 +269,9 @@ impl KvStore for SledStore { self.search_trigrams .apply_batch(batch_search_trigrams) .map_err(|e| format!("Failed to apply search_trigrams batch: {}", e))?; + self.envelopes + .apply_batch(batch_envelopes) + .map_err(|e| format!("Failed to apply envelopes batch: {}", e))?; Ok(()) } diff --git a/lib/src/db/trees.rs b/lib/src/db/trees.rs index cf02ffc9f..b71161d85 100644 --- a/lib/src/db/trees.rs +++ b/lib/src/db/trees.rs @@ -35,6 +35,12 @@ pub enum Tree { SearchDocTokens, /// Trigram → term map for 1-edit candidate generation on longer tokens. SearchTrigrams, + /// Signed commit envelopes kept per resource (the audit floor, F6/F7 in + /// `planning/completed/commit-retention-floor-decision.md`). Key: + /// `pure_id || 0x00 || createdAt (u64 BE) || 0x00 || signature`, value: the + /// commit's JSON-AD exactly as accepted. Not a resource, not indexed, so + /// it never shows up in queries or `all_resources`. See `crate::envelopes`. + Envelopes, } const RESOURCES: &str = "resources_v3"; @@ -61,6 +67,7 @@ const SEARCH_POSTINGS: &str = "search_postings_v1"; const SEARCH_DOCS: &str = "search_docs_v1"; const SEARCH_DOC_TOKENS: &str = "search_doc_tokens_v1"; const SEARCH_TRIGRAMS: &str = "search_trigrams_v1"; +const ENVELOPES: &str = "envelopes_v1"; impl std::fmt::Display for Tree { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -79,6 +86,7 @@ impl std::fmt::Display for Tree { Tree::SearchDocs => f.write_str(SEARCH_DOCS), Tree::SearchDocTokens => f.write_str(SEARCH_DOC_TOKENS), Tree::SearchTrigrams => f.write_str(SEARCH_TRIGRAMS), + Tree::Envelopes => f.write_str(ENVELOPES), } } } @@ -101,6 +109,7 @@ impl AsRef<[u8]> for Tree { Tree::SearchDocs => SEARCH_DOCS.as_bytes(), Tree::SearchDocTokens => SEARCH_DOC_TOKENS.as_bytes(), Tree::SearchTrigrams => SEARCH_TRIGRAMS.as_bytes(), + Tree::Envelopes => ENVELOPES.as_bytes(), } } } diff --git a/lib/src/envelopes.rs b/lib/src/envelopes.rs new file mode 100644 index 000000000..f8d67bcb0 --- /dev/null +++ b/lib/src/envelopes.rs @@ -0,0 +1,615 @@ +//! Signed commit envelopes kept per resource. +//! +//! Authorization is decided on state (`read` / `write` / `parent` in the +//! projection); the Loro oplog is the history of *what* changed and when. What +//! neither carries is *who signed the state you are looking at*: the oplog's +//! change messages are opaque drain tokens, and `lastCommit` is only an id. +//! This tree keeps the signed JSON-AD of the commits that produced a resource, +//! so any node holding it can re-verify the signature and attribute the state, +//! offline. See `planning/completed/commit-retention-floor-decision.md` +//! (F6 latest envelope, F7 every envelope). +//! +//! Layout ([`Tree::Envelopes`]): key +//! `pure_id || 0x00 || createdAt (u64 BE) || 0x00 || signature`, value the +//! commit JSON-AD exactly as `/commit` or the `COMMIT` frame accepted it. A +//! prefix scan on the pure id lists a resource's envelopes in time order. The +//! rows are not resources and not indexed: they never show up in queries, +//! `all_resources`, search or collections, so nothing has to filter +//! `did:ad:commit:` subjects by hand. +//! +//! How many rows survive is [`EnvelopeRetention`]: `Latest` keeps the one +//! that produced the current state (the floor), `All` keeps every envelope +//! and turns the oplog into a signed audit log ([`attribute_history`]). +//! +//! What is deliberately not here: envelopes inside the Loro doc (an envelope +//! would then sign a document containing itself), and a retention schedule +//! beyond the two settings. Replication of these rows is the sync layer's +//! job (`planning/auditability-loro-history.md`). + +use crate::db::trees::{Method, Operation, Transaction, Tree}; +use crate::errors::AtomicResult; +use crate::{commit::CommitResponse, Db}; + +/// Which envelopes a node keeps per resource. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum EnvelopeRetention { + /// One row per resource: the envelope that produced the current state. + /// Attribution of the current state stays verifiable; older edits are + /// visible in the Loro oplog but unattributed. + #[default] + Latest, + /// Every envelope. Each Loro change maps back to the signed commit that + /// introduced it, so History can show a verified signer per version. + All, +} + +impl EnvelopeRetention { + pub fn parse(value: &str) -> Option<Self> { + match value.trim().to_ascii_lowercase().as_str() { + "latest" => Some(Self::Latest), + "all" | "full" => Some(Self::All), + _ => None, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Latest => "latest", + Self::All => "all", + } + } +} + +/// One retained envelope, decoded from its key. `json` is the signed body. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct StoredEnvelope { + /// Pure id of the resource the commit is about. + pub subject: String, + /// Commit `createdAt`, Unix milliseconds. + pub created_at: i64, + pub signature: String, + /// The commit's JSON-AD exactly as accepted. + pub json: String, +} + +impl StoredEnvelope { + /// The commit id this envelope is stored under (`did:ad:commit:<sig>`), + /// the same value `lastCommit` stamps on the resource. + pub fn commit_id(&self) -> String { + format!("did:ad:commit:{}", self.signature) + } + + /// Whether this envelope is a destroy. + pub fn is_destroy(&self) -> bool { + serde_json::from_str::<serde_json::Value>(&self.json) + .ok() + .and_then(|v| v.get(crate::urls::DESTROY).and_then(|d| d.as_bool())) + .unwrap_or(false) + } +} + +fn prefix(subject: &str) -> Vec<u8> { + let pure = crate::Subject::from_raw(subject, None).pure_id(); + let mut key = Vec::with_capacity(pure.len() + 1); + key.extend_from_slice(pure.as_bytes()); + key.push(0); + key +} + +fn key(subject: &str, created_at: i64, signature: &str) -> Vec<u8> { + let mut key = prefix(subject); + key.extend_from_slice(&(created_at.max(0) as u64).to_be_bytes()); + key.push(0); + key.extend_from_slice(signature.as_bytes()); + key +} + +fn decode(key: &[u8], value: Vec<u8>) -> Option<StoredEnvelope> { + let subject_end = key.iter().position(|b| *b == 0)?; + let subject = std::str::from_utf8(&key[..subject_end]).ok()?.to_string(); + let rest = &key[subject_end + 1..]; + if rest.len() < 9 || rest[8] != 0 { + return None; + } + let created_at = u64::from_be_bytes(rest[..8].try_into().ok()?) as i64; + let signature = std::str::from_utf8(&rest[9..]).ok()?.to_string(); + let json = String::from_utf8(value).ok()?; + Some(StoredEnvelope { + subject, + created_at, + signature, + json, + }) +} + +/// Queue the writes that keep this commit's envelope, honouring the store's +/// retention. Appended to the apply transaction so the envelope lands with +/// the state it signs, or not at all. Unsigned commits (internal writes) +/// have nothing to keep. +pub fn record_ops( + store: &Db, + response: &CommitResponse, + transaction: &mut Transaction, +) -> AtomicResult<()> { + let Some(signature) = response.commit.signature.as_deref() else { + return Ok(()); + }; + let subject = response.commit.subject.as_str(); + let json = response.commit_resource.to_json_ad(None)?; + let new_key = key(subject, response.commit.created_at, signature); + + if store.envelope_retention() == EnvelopeRetention::Latest { + for existing in store.kv.scan_prefix(Tree::Envelopes, &prefix(subject)) { + let (old_key, _) = existing?; + if old_key != new_key { + transaction.push(Operation { + tree: Tree::Envelopes, + method: Method::Delete, + key: old_key, + val: None, + }); + } + } + } + + transaction.push(Operation { + tree: Tree::Envelopes, + method: Method::Insert, + key: new_key, + val: Some(json.into_bytes()), + }); + Ok(()) +} + +/// Every retained envelope of a resource, oldest first. +pub fn envelopes(store: &Db, subject: &str) -> Vec<StoredEnvelope> { + store + .kv + .scan_prefix(Tree::Envelopes, &prefix(subject)) + .filter_map(|entry| entry.ok()) + .filter_map(|(k, v)| decode(&k, v)) + .collect() +} + +/// The envelope that produced the resource's current state, if kept. +pub fn latest_envelope(store: &Db, subject: &str) -> Option<StoredEnvelope> { + envelopes(store, subject).into_iter().last() +} + +/// Drop every retained envelope of a resource. Not called on destroy: the +/// destroy envelope is the proof a peer needs (`SYNC_DIFF.removeCommits`). +pub fn clear_envelopes(store: &Db, subject: &str) { + for (k, _) in store + .kv + .scan_prefix(Tree::Envelopes, &prefix(subject)) + .flatten() + { + let _ = store.kv.remove(Tree::Envelopes, &k); + } +} + +/// The genesis change's message is the creator's agent subject (written by +/// the browser and by `Commit::create_did`), which `createdBy` reads. +fn is_genesis_carrier(token: &str) -> bool { + token.starts_with("did:ad:agent:") +} + +/// One signed change, as History shows it. +#[derive(Debug, Clone, serde::Serialize)] +pub struct Attribution { + pub signer: String, + /// Commit `createdAt`, Unix milliseconds. + pub created_at: i64, + pub signature: String, + /// The signature checks out against the signer's key on this node. + pub verified: bool, + /// Loro change messages (the client's drain tokens) the envelope's update + /// introduced. History buckets versions by the same token, so a version + /// maps to its signer by lookup. + pub tokens: Vec<String>, + pub destroy: bool, + pub genesis: bool, +} + +/// What this node can say about who signed a resource's history. +#[derive(Debug, Clone, serde::Serialize)] +pub struct HistoryAttribution { + pub subject: String, + /// Retention this node runs; tells a reader whether missing attributions + /// are a gap or a policy. + pub retention: &'static str, + /// Oldest first. + pub attributions: Vec<Attribution>, + /// Every client-authored change in the stored oplog (a change carrying + /// a drain token) is claimed by a verified envelope. Server bookkeeping + /// (the `lastCommit` stamp, derived `drive`) writes untokened changes and + /// is not counted. `false` while the subject is destroyed or nothing is + /// retained. + pub complete: bool, +} + +/// Verify the retained envelopes of a resource and map them onto its Loro +/// history. Each envelope's signature is checked with the same code apply +/// uses, and its `loroUpdate` is imported into a fresh doc to read which +/// change tokens it introduced. `complete` is whether every tokened change +/// in the stored oplog is claimed by a verified envelope. Anything not +/// covered is unattributed, never a guessed signer. +pub async fn attribute_history(store: &Db, subject: &str) -> AtomicResult<HistoryAttribution> { + let retention = store.envelope_retention().as_str(); + let mut attributions: Vec<Attribution> = Vec::new(); + + for envelope in envelopes(store, subject) { + let resource = crate::parse::parse_json_ad_commit_resource(&envelope.json, store).await?; + let commit = crate::commit::Commit::from_resource(resource)?; + let verified = commit.validate_signature(store).await.is_ok(); + + // Tokens this envelope introduced. A genesis carries a snapshot and + // a browser edit only its delta, but a Rust builder commit (and a + // client re-exporting from an older cursor) repeats earlier changes; + // a token is credited to the first retained envelope that carried + // it, so each change has one signer. The genesis change's message is + // the creator's subject and is proven by the inline genesis + // certificate, not by whoever later shipped a snapshot containing + // it: only a genesis envelope may claim it. + let is_genesis = commit.is_genesis == Some(true); + let mut tokens = Vec::new(); + if let Some(update) = commit.loro_update.as_deref() { + let probe = crate::loro::AtomicLoroDoc::new(); + if probe.import_update(update).is_ok() { + for change in probe.get_history() { + if let Some(message) = change.message { + if is_genesis_carrier(&message) && !is_genesis { + continue; + } + let claimed = attributions.iter().any(|a| a.tokens.contains(&message)); + if !claimed && !tokens.contains(&message) { + tokens.push(message); + } + } + } + } + } + + attributions.push(Attribution { + signer: commit.signer.to_string(), + created_at: commit.created_at, + signature: envelope.signature.clone(), + verified, + tokens, + destroy: commit.destroy.unwrap_or(false), + genesis: is_genesis, + }); + } + + let pure = crate::Subject::from_raw(subject, None).pure_id(); + let stored_tokens: Option<Vec<String>> = store + .kv + .get(Tree::LoroSnapshots, pure.as_bytes()) + .ok() + .flatten() + .and_then(|bytes| crate::loro::AtomicLoroDoc::from_snapshot(&bytes).ok()) + .map(|doc| { + doc.get_history() + .into_iter() + .filter_map(|change| change.message) + .collect() + }); + // The genesis change is covered by the resource's inline certificate + // (F1), so it is not required here; every other tokened change must be. + let complete = match stored_tokens { + Some(tokens) => { + !attributions.is_empty() + && tokens + .iter() + .filter(|token| !is_genesis_carrier(token)) + .all(|token| { + attributions + .iter() + .any(|a| a.verified && a.tokens.contains(token)) + }) + } + None => false, + }; + + Ok(HistoryAttribution { + subject: pure, + retention, + attributions, + complete, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::agents::ForAgent; + use crate::sync::engine::{ingest_commit_json, CommitIngestOpts}; + use crate::{urls, Storelike, Value}; + + /// A signed content edit by the store's default agent, applied through + /// `Db::apply_commit` like every other write. + async fn signed_edit(db: &Db, subject: &crate::Subject, name: &str) { + let mut resource = db.get_resource(subject).await.unwrap(); + resource + .set(urls::NAME.into(), Value::String(name.into()), db) + .await + .unwrap(); + let response = resource.save_locally(db).await.unwrap(); + assert!(response.commit.signature.is_some(), "save_locally signs"); + } + + async fn child(db: &Db, drive: &str) -> crate::Subject { + let subject = db + .create_resource( + urls::CLASS, + drive, + "Doc", + Some(vec![ + (urls::DESCRIPTION, Value::String("d".into())), + (urls::SHORTNAME, Value::Slug("doc".into())), + ]), + ) + .await + .unwrap(); + crate::Subject::from_raw(&subject, None) + } + + #[tokio::test] + async fn latest_retention_keeps_one_envelope_per_resource() { + let db = Db::init_temp("envelopes_latest").await.unwrap(); + let (_alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + assert_eq!( + envelopes(&db, subject.as_str()).len(), + 1, + "create_resource signs a genesis" + ); + + for name in ["one", "two"] { + signed_edit(&db, &subject, name).await; + } + let kept = envelopes(&db, subject.as_str()); + assert_eq!(kept.len(), 1, "Latest keeps only the newest envelope"); + let latest = latest_envelope(&db, subject.as_str()).unwrap(); + assert_eq!(kept[0], latest); + let stamp = db + .get_resource(&subject) + .await + .unwrap() + .get(urls::LAST_COMMIT) + .unwrap() + .to_string(); + assert_eq!(latest.commit_id(), stamp, "the kept envelope is lastCommit"); + } + + #[tokio::test] + async fn all_retention_keeps_every_envelope_in_time_order() { + let db = Db::init_temp("envelopes_all").await.unwrap(); + db.set_envelope_retention(EnvelopeRetention::All); + let (_alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + + for name in ["one", "two", "three"] { + signed_edit(&db, &subject, name).await; + } + let kept = envelopes(&db, subject.as_str()); + assert_eq!(kept.len(), 4, "genesis plus three edits"); + assert!(kept.windows(2).all(|w| w[0].created_at <= w[1].created_at)); + let stamp = db + .get_resource(&subject) + .await + .unwrap() + .get(urls::LAST_COMMIT) + .unwrap() + .to_string(); + assert_eq!(kept.last().unwrap().commit_id(), stamp); + } + + #[tokio::test] + async fn envelopes_are_not_resources_or_query_hits() { + let db = Db::init_temp("envelopes_not_indexed").await.unwrap(); + let (_alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + let latest = latest_envelope(&db, subject.as_str()).unwrap(); + // The genesis commit row is retained as a resource (critical), but the + // envelope tree itself is invisible to the resource model. + assert!(!db.has_resource_locally(&format!("envelope:{}", latest.signature))); + let mut query = crate::storelike::Query::new_prop_val(urls::SIGNER, "did:ad:agent:nobody"); + query.limit = Some(10); + assert_eq!(db.query(&query).await.unwrap().count, 0); + } + + #[tokio::test] + async fn history_attribution_maps_verified_signers_onto_loro_tokens() { + let db = Db::init_temp("envelopes_attribution").await.unwrap(); + db.set_envelope_retention(EnvelopeRetention::All); + let (alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + signed_edit(&db, &subject, "edited").await; + + let report = attribute_history(&db, subject.as_str()).await.unwrap(); + assert_eq!(report.retention, "all"); + assert_eq!(report.attributions.len(), 2); + assert!(report.attributions.iter().all(|a| a.verified)); + assert!(report.attributions[0].genesis); + assert!(!report.attributions[1].genesis); + assert!(report + .attributions + .iter() + .all(|a| a.signer == alice.subject)); + assert!( + report.complete, + "replaying the retained envelopes must reproduce the stored oplog" + ); + + // Every Loro change of the stored doc is claimed by exactly one envelope. + let resource = db.get_resource(&subject).await.unwrap(); + let versions = crate::history::versions(&resource).unwrap(); + for version in versions.iter().filter_map(|v| v.message.clone()) { + let owners = report + .attributions + .iter() + .filter(|a| a.tokens.contains(&version)) + .count(); + assert_eq!(owners, 1, "token {version} must map to one signer"); + } + } + + #[tokio::test] + async fn tampered_envelope_is_unverified_and_history_incomplete() { + let db = Db::init_temp("envelopes_tampered").await.unwrap(); + db.set_envelope_retention(EnvelopeRetention::All); + let (_alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + signed_edit(&db, &subject, "edited").await; + + // Corrupt the newest stored row in place. + let rows = envelopes(&db, subject.as_str()); + let last = rows.last().unwrap(); + let mut broken: serde_json::Value = serde_json::from_str(&last.json).unwrap(); + broken[urls::SIGNATURE] = serde_json::Value::String("AAAA".into()); + db.kv + .insert( + Tree::Envelopes, + &key(subject.as_str(), last.created_at, &last.signature), + broken.to_string().as_bytes(), + ) + .unwrap(); + + let report = attribute_history(&db, subject.as_str()).await.unwrap(); + assert!(report.attributions[0].verified); + assert!(!report.attributions[1].verified); + assert!(!report.complete); + } + + #[tokio::test] + async fn latest_retention_under_a_second_writer_keeps_the_newest_signer() { + let db = Db::init_temp("envelopes_two_writers").await.unwrap(); + let (alice, drive) = db.setup("Alice").await.unwrap(); + let bob = db.create_agent(Some("Bob")).await.unwrap(); + let subject = child(&db, &drive).await; + let mut resource = db.get_resource(&subject).await.unwrap(); + resource + .set_unsafe( + urls::WRITE.into(), + Value::ResourceArray(vec![ + alice.subject.to_string().into(), + bob.subject.to_string().into(), + ]), + ) + .unwrap(); + db.add_resource_opts(&resource, false, true, true) + .await + .unwrap(); + + db.set_default_agent(bob.clone()); + signed_edit(&db, &subject, "by bob").await; + db.set_default_agent(alice.clone()); + let report = attribute_history(&db, subject.as_str()).await.unwrap(); + assert_eq!(report.attributions.len(), 1); + assert_eq!(report.attributions[0].signer, bob.subject.to_string()); + assert!(report.attributions[0].verified); + assert!( + !report.attributions[0] + .tokens + .iter() + .any(|t| t.starts_with("did:ad:agent:")), + "a snapshot-carrying edit must not be credited with the genesis change" + ); + assert!( + report.complete, + "the genesis is proven by its certificate; the only other signed change is Bob's" + ); + } + + #[tokio::test] + async fn all_retention_credits_each_writer_with_their_own_change() { + let db = Db::init_temp("envelopes_two_writers_all").await.unwrap(); + db.set_envelope_retention(EnvelopeRetention::All); + let (alice, drive) = db.setup("Alice").await.unwrap(); + let bob = db.create_agent(Some("Bob")).await.unwrap(); + let subject = child(&db, &drive).await; + let mut resource = db.get_resource(&subject).await.unwrap(); + resource + .set_unsafe( + urls::WRITE.into(), + Value::ResourceArray(vec![ + alice.subject.to_string().into(), + bob.subject.to_string().into(), + ]), + ) + .unwrap(); + db.add_resource_opts(&resource, false, true, true) + .await + .unwrap(); + + signed_edit(&db, &subject, "by alice").await; + db.set_default_agent(bob.clone()); + signed_edit(&db, &subject, "by bob").await; + db.set_default_agent(alice.clone()); + + let report = attribute_history(&db, subject.as_str()).await.unwrap(); + assert!(report.complete); + let signers: Vec<&str> = report + .attributions + .iter() + .map(|a| a.signer.as_str()) + .collect(); + assert_eq!( + signers, + vec![ + alice.subject.as_str(), + alice.subject.as_str(), + bob.subject.as_str() + ], + "genesis, Alice's edit, Bob's edit" + ); + assert!(report.attributions[0].genesis); + assert_eq!(report.attributions[1].tokens.len(), 1); + assert_eq!(report.attributions[2].tokens.len(), 1); + assert_ne!(report.attributions[1].tokens, report.attributions[2].tokens); + let stored = db.get_resource(&subject).await.unwrap(); + let versions = crate::history::versions(&stored).unwrap(); + for token in versions.iter().filter_map(|v| v.message.clone()) { + let owners = report + .attributions + .iter() + .filter(|a| a.tokens.contains(&token)) + .count(); + assert_eq!(owners, 1, "token {token} must map to exactly one signer"); + } + } + + #[tokio::test] + async fn destroy_envelope_is_the_latest_row_of_a_destroyed_subject() { + let db = Db::init_temp("envelopes_destroy").await.unwrap(); + let (alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + let resource = db.get_resource(&subject).await.unwrap(); + let mut builder = crate::commit::CommitBuilder::new(subject.clone()); + builder.destroy(true); + let commit = builder.sign(&alice, &db, &resource).await.unwrap(); + let json = commit + .into_resource(&db) + .await + .unwrap() + .to_json_ad(None) + .unwrap(); + ingest_commit_json(&db, &json, &CommitIngestOpts::peer()) + .await + .unwrap(); + + let latest = latest_envelope(&db, subject.as_str()).unwrap(); + assert!(latest.is_destroy()); + assert_eq!( + crate::sync::tombstones::destroy_envelope(&db, subject.as_str()).as_deref(), + Some(latest.json.as_str()), + "the tombstone's envelope is the envelope tree's latest row" + ); + assert!(crate::sync::tombstones::is_tombstoned( + &db, + subject.as_str() + )); + let _ = ForAgent::Public; + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 1eccfe30e..9897c9720 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -80,6 +80,8 @@ pub mod db; pub mod discovery; #[cfg(feature = "db")] pub mod endpoints; +#[cfg(feature = "db")] +pub mod envelopes; pub mod errors; pub mod expression; pub mod genesis; diff --git a/lib/src/resources.rs b/lib/src/resources.rs index 8c022cc4c..0232ab6e4 100644 --- a/lib/src/resources.rs +++ b/lib/src/resources.rs @@ -1074,6 +1074,16 @@ impl Resource { let Some(doc) = self.loro.as_ref() else { return Ok(()); }; + // Tag the pending edits as one change with a unique token, exactly as + // the browser does before signing: history buckets versions by this + // message, and the signed envelope that carries the change is matched + // back to it by the same token (`crate::envelopes::attribute_history`). + // A no-op when nothing is pending. + doc.commit_with_message(&format!( + "c-{:x}-{}", + crate::utils::now(), + crate::utils::random_string(6) + )); let base = match self.get(urls::LORO_UPDATE) { Ok(Value::LoroDoc(snapshot)) => Some(snapshot.clone()), _ => None, diff --git a/lib/src/sync/tombstones.rs b/lib/src/sync/tombstones.rs index 27b52d8d7..e8c3e4423 100644 --- a/lib/src/sync/tombstones.rs +++ b/lib/src/sync/tombstones.rs @@ -1,20 +1,17 @@ //! Tombstones for resources destroyed locally. Used during Iroh/WS bulk sync so //! peers delete instead of re-uploading or resurrecting deleted subjects. //! -//! The value is either a one-byte marker (`[1]`) for an unsigned tombstone -//! (cascade delete, cache eviction, a peer that sent `remove[]` without an -//! envelope) or the signed destroy commit's JSON-AD. The latter is what -//! `SYNC_DIFF.removeCommits` carries so a replica can apply the destroy -//! with the same signature + rights check as a live `COMMIT`. Full -//! envelope-on-resource storage (`Tree::Envelopes`) is still the commit -//! retention floor; this is destroy-only evidence on the existing -//! `PluginMeta` tombstone key. +//! A tombstone is a one-byte marker on the `PluginMeta` tree. The *signed* +//! destroy commit, when there is one, is not stored here: it is the subject's +//! latest row in [`crate::envelopes`] (`Tree::Envelopes`), which +//! `apply_commit` writes for every signed commit. [`destroy_envelope`] reads +//! it from there so `SYNC_DIFF.removeCommits` can carry the same envelope the +//! live `COMMIT` path forwards. use crate::db::trees::Tree; use crate::Db; const PREFIX: &[u8] = b"tombstone:"; -const UNSIGNED_MARKER: &[u8] = &[1]; fn tombstone_key(subject: &str) -> Vec<u8> { let pure = crate::Subject::from_raw(subject, None).pure_id(); @@ -25,39 +22,22 @@ fn tombstone_key(subject: &str) -> Vec<u8> { } /// Remember that this subject was intentionally destroyed on this device. -/// Does not overwrite a stored destroy envelope — a later unsigned path -/// (cascade, `apply_destroy`) must not drop the signed evidence. pub fn record_tombstone(store: &Db, subject: &str) { - if destroy_envelope(store, subject).is_some() { - return; - } let key = tombstone_key(subject); - let _ = store.kv.insert(Tree::PluginMeta, &key, UNSIGNED_MARKER); + let _ = store.kv.insert(Tree::PluginMeta, &key, &[1]); } -/// Store (or replace) the signed destroy commit that authorises this -/// tombstone. Overwrites an unsigned marker. -pub fn record_destroy_envelope(store: &Db, subject: &str, commit_json: &str) { - if commit_json.is_empty() { - record_tombstone(store, subject); - return; - } - let key = tombstone_key(subject); - let _ = store - .kv - .insert(Tree::PluginMeta, &key, commit_json.as_bytes()); -} - -/// The signed destroy commit JSON stored with this tombstone, if any. -/// `None` for an unsigned marker or a missing tombstone. +/// The signed destroy commit JSON for this tombstone, if this node kept it: +/// the subject's latest envelope, when that envelope is a destroy. `None` +/// for an unsigned tombstone (cascade delete, cache eviction, a peer that +/// sent `remove[]` without an envelope) or a missing one. pub fn destroy_envelope(store: &Db, subject: &str) -> Option<String> { - let key = tombstone_key(subject); - let bytes = store.kv.get(Tree::PluginMeta, &key).ok().flatten()?; - if bytes.is_empty() || bytes == UNSIGNED_MARKER { + if !is_tombstoned(store, subject) { return None; } - let json = String::from_utf8(bytes).ok()?; - json.starts_with('{').then_some(json) + crate::envelopes::latest_envelope(store, subject) + .filter(|envelope| envelope.is_destroy()) + .map(|envelope| envelope.json) } /// True if we previously destroyed this subject here (do not re-import from peers). @@ -73,11 +53,11 @@ pub fn is_tombstoned(store: &Db, subject: &str) -> bool { /// Clear a tombstone — the subject was legitimately re-created (F11, /// planning/unified-sync.md). A tombstone only means "don't resurrect this -/// deleted subject"; once a rights-checked genesis commit re-creates it, -/// that invariant is stale and must not keep suppressing it from future -/// bulk-sync imports (`is_tombstoned` gates `import_sync_push` and the -/// `SYNC_VV` remove-list) or the newly-recreated resource silently never -/// reaches other replicas. No-op if there was no tombstone to clear. +/// deleted subject"; once a genesis commit re-creates it, that invariant is +/// stale and must not keep suppressing it from future bulk-sync imports +/// (`is_tombstoned` gates `import_sync_push` and the `SYNC_VV` remove-list) +/// or the newly-recreated resource silently never reaches other replicas. +/// No-op if there was no tombstone to clear. pub fn clear_tombstone(store: &Db, subject: &str) { let key = tombstone_key(subject); let _ = store.kv.remove(Tree::PluginMeta, &key); @@ -133,20 +113,4 @@ mod key_normalization_tests { assert!(is_tombstoned(&db, subject)); assert_eq!(destroy_envelope(&db, subject), None); } - - #[tokio::test] - async fn destroy_envelope_round_trips_and_survives_unsigned_rerecord() { - let db = Db::init_temp("tombstone_envelope_roundtrip").await.unwrap(); - let subject = "did:ad:example"; - let json = r#"{"https://atomicdata.dev/properties/destroy":true}"#; - record_destroy_envelope(&db, subject, json); - assert!(is_tombstoned(&db, subject)); - assert_eq!(destroy_envelope(&db, subject).as_deref(), Some(json)); - record_tombstone(&db, subject); - assert_eq!( - destroy_envelope(&db, subject).as_deref(), - Some(json), - "an unsigned rerecord must not drop a stored destroy envelope" - ); - } } diff --git a/planning/README.md b/planning/README.md index b961b2203..2f5fb969e 100644 --- a/planning/README.md +++ b/planning/README.md @@ -25,7 +25,7 @@ now live in [`completed/`](./completed/): - [`runtime-boundary-decision.md`](./completed/runtime-boundary-decision.md) — `AtomicNode` in `lib/src/runtime/` is the binding runtime; no parallel `simple.rs` / `ffi/`. - [`authority-unit-decision.md`](./completed/authority-unit-decision.md) — the drive stays the unit of authority; the zone chain is hybrid/additive. -- [`commit-retention-floor-decision.md`](./completed/commit-retention-floor-decision.md) — envelope-on-resource; #1313 waits for `Tree::Envelopes`. +- [`commit-retention-floor-decision.md`](./completed/commit-retention-floor-decision.md) — envelope-on-resource; amended 2026-09-05, `Tree::Envelopes` ships in #1313. - [`trust-model-decision.md`](./completed/trust-model-decision.md) — the node that owns the URL is trusted with plaintext; anything that only stores is blind. - [`schema-routes-decision.md`](./completed/schema-routes-decision.md) — `did:ad:frozen` is the on-ramp, optional schema is the write-path policy. @@ -60,7 +60,7 @@ Remaining work, not "this file exists." | [`index-performance.md`](./index-performance.md) | First tranche shipped. Structural permission-check fix is `zones.md`, not built. | | [`disk-storage-and-persistence-optimization.md`](./disk-storage-and-persistence-optimization.md) | **Proposal.** Full-snapshot writes, no auto-compaction, O(file) open fsync. | | [`virtual-drive.md`](./virtual-drive.md) | **Shipped** as a local NFS mount in the Tauri desktop app (`desktop/src/vfs.rs`). Still proposal: headless-server mount, FUSE/WinFSP, native cloud-sync APIs, mobile providers. | -| [`commit-retention-and-state-certificates.md`](./commit-retention-and-state-certificates.md) | **Proposal.** Commits stay signed write certificates; retention is node policy. Content-commit drop shipped; remaining is optional audit retention. | +| [`commit-retention-and-state-certificates.md`](./commit-retention-and-state-certificates.md) | **Mostly shipped.** Commits are signed envelopes; content rows dropped; `Tree::Envelopes` keeps the latest (or all) per resource. Remaining: envelope carriage in bulk sync and the vault. | | [`p2p-presence.md`](./p2p-presence.md) | **Mostly built.** `EPHEMERAL 0x40` codec, peer send/receive and the server bridge are in (`lib/src/sync/iroh_e2e.rs` `e2e_presence_crosses_the_link_without_being_stored`). Remaining: two-device verification (M12), bandwidth measurement (OQ1). Scoped to your own devices by product choice. | | [`reticulum-sync.md`](./reticulum-sync.md) | **Proposal.** Atomic sync protocol over Reticulum. | | [`json-schema-code-first.md`](./json-schema-code-first.md) | **Proposal**; `defineSchema` + frozen `did:ad:` schemas in flight in PR #1262 (not on `develop`). Code-first JSON Schema → local DID-backed Class/Property resources. | @@ -90,7 +90,7 @@ Not top-level plans. Indexed so they do not go missing. | [`main-drive-and-paths.md`](./main-drive-and-paths.md) | Strategy. DID-branch deployment: root drive, legacy URLs, human-readable paths. | | [`actions.md`](./actions.md) | **Steps 1–4 shipped.** Registry drives ⌘M, ⌘K (capped prefix match), hotkeys, the shortcuts overlay/page, and simple AI tools. Remaining: MCP projection when a server exists. | | [`silent-failures.md`](./silent-failures.md) | Living log of error-handling failures that reported success (2026-08-21). Carries M8 from the pairing field test. | -| [`auditability-loro-history.md`](./auditability-loro-history.md) | Open. History = verifiable log for every replica, including new users (`git clone`). Envelopes live on the resource and catch-up must copy them; not a `/commits` class. | +| [`auditability-loro-history.md`](./auditability-loro-history.md) | **Building.** `Tree::Envelopes` + `attribute_history` + `/history-attribution` + History Verified badge shipped 2026-09-05. Next: envelopes travel in bulk sync and the vault. | Closed decisions, as-built records, closed explorations and fixed notes live in [`completed/`](./completed/): the five decisions above, the 2026-07 sync diff --git a/planning/auditability-loro-history.md b/planning/auditability-loro-history.md index 563027446..ff21d3f06 100644 --- a/planning/auditability-loro-history.md +++ b/planning/auditability-loro-history.md @@ -1,6 +1,8 @@ # Verifiable History -> **Status:** Open (2026-08-27). Companion to +> **Status:** Building (2026-09-05). Decided in +> [`completed/commit-retention-floor-decision.md`](./completed/commit-retention-floor-decision.md) +> (option C, amended 2026-09-05). Companion to > [`commit-retention-and-state-certificates.md`](./commit-retention-and-state-certificates.md) > and [`authorization-sync.md`](./authorization-sync.md). > @@ -23,131 +25,118 @@ Each History row: | **Who** | Envelope `signer` | Ed25519 over that JSON | | **Proof** | Envelope `signature` | Same bytes `/commit` accepted | -Today History is Loro-only (`Edited … by peer {hex}`). Content envelopes -are discarded after apply. A clone gets the document, not the log. - -## Git analogy - -| Git | Atomic | -| --- | --- | -| Blob / tree | Loro snapshot + oplog | -| Commit object (author, time, tree hash, signature if signed) | Signed envelope | -| `git clone` copies objects | Catch-up must copy envelopes **with** the snapshot | -| `git log` | History page | - -Snapshot-only `SYNC_PUSH` is `git clone --no-checkout` of the tree with -the `.git` directory empty. Unattributed History on a new device is that -failure mode. It is not an acceptable default. - -A linear `previousCommit` chain is **not** the git part we need (git -also has merge commits; Loro already merges). The git part we need is -**replicated commit objects**. - ## Split ```text -Loro oplog = mergeable document (what) -Signed envelope = commit object (who, when, proof) +Loro oplog = mergeable document (what, when, which peer typed) +Signed envelope = commit object (who signed, when, proof) Both = replicate together Graph / /commits = not a history store ``` -Do not treat Loro change messages as “who.” They are plaintext. -Do not restore `/commits` as a queryable class. The log belongs to the -resource, like git objects belong to the repo — not to a site-wide -commit collection. - -## Requirement: proofs travel with the resource - -After apply, keep the signed envelope **on the resource’s replica -state**, so the next `SYNC_PUSH` / OPFS snapshot / Iroh catch-up -includes it. - -Preferred: a sibling Loro container on the same doc (e.g. `envelopes`: -commit-id → signed JSON-AD). Then today’s snapshot sync *is* clone of -the log. No `/commits` class, no extra Layer 2 trailer, no “blob table -the new user never sees.” - -Apply: - -1. Verify envelope, import `loroUpdate` into `properties` (as now). -2. Append the signed envelope to `envelopes` (CRDT map/list — concurrent - writers both land). -3. Stamp `lastCommit` for echo-dedup / genesis detection (as now). - -History: - -```text -Loro version → lastCommit / envelopes key → verify signature - → checkout → diff / restore -``` - -Missing envelope ⇒ **Unattributed** (legacy snapshot, truncated replica, -tamper). That is an error state, not the path for a new invitee. - -Authorization-critical commits (genesis / ACL / parent / destroy) stay -in the graph as the must-retain floor. Content envelopes live on the -resource. Neither is a site-wide event log. - -## Storage - -Same `Db` as the document (server redb, browser OPFS), because they are -part of that resource’s replica, not a server-only audit tape. - -If they live **in** the Loro doc, `Tree::LoroSnapshots` already holds -them. If they live in a side tree, bulk sync **must** send that tree -with the snapshot — same requirement, more wire. Prefer in-doc so -catch-up cannot forget them. - -Cost: the envelope repeats `loroUpdate`. That duplication is the -verifiable object, as a git commit repeats a pointer at content. A -header-only object (signer, createdAt, signature, hash of the change) -is a later size win; v1 stores the full signed body so verify matches -today’s `/commit` bytes. - -## Sync - -| Path | Must happen | -| --- | --- | -| Live `COMMIT` | Apply + persist envelope on the receiver (stop dropping it). | -| Bulk `SYNC_PUSH` | Snapshot includes envelopes (in-doc) **or** the push is incomplete. A new user who only got `properties` has an unverifiable log. | -| Offline local | OPFS snapshot includes envelopes; History verifies without network. | - -Layer 2 that imports a snapshot and ignores `envelopes` is a bug, not a -mode. See unsigned `SYNC_PUSH` in -[`authorization-sync.md`](./authorization-sync.md). - -## History row (target UI) - -- **Verified** — `{agent name} · {createdAt}` after Ed25519 check -- **Unattributed** — warning, not the normal row (legacy / incomplete replica) -- Diff and restore stay Loro checkouts -- No navigation to `did:ad:commit:…` as a document +Do not treat Loro change messages as “who.” They are plaintext. Do not +restore `/commits` as a queryable class. The log belongs to the resource, +like git objects belong to the repo, not to a site-wide commit collection. + +## What is built (2026-09-05) + +**Storage.** `Tree::Envelopes` (`lib/src/envelopes.rs`, all KV backends). +Key `pure_id ‖ 0x00 ‖ createdAt (u64 BE) ‖ 0x00 ‖ signature`, value the +commit JSON-AD exactly as `/commit` or the `COMMIT` frame accepted it. A +prefix scan on the pure id lists a resource's envelopes in time order. Not +a resource, not indexed: never in queries, `all_resources`, search or +collections, so nothing filters `did:ad:commit:` subjects by hand. + +**Write.** `Db::apply_commit` queues the row in the same transaction as +the state it signs (`envelopes::record_ops`). Every signed commit, every +ingest path, one place. Critical commits (genesis, rights, parent, +destroy) additionally keep their `Tree::Resources` row as before, for +`AuthorizationProof` (P3) to find by `subject`. The destroy envelope that +#1370 put on the tombstone value is now just the subject's latest row; +`tombstones::destroy_envelope` reads it from there. + +**Retention.** `EnvelopeRetention::{Latest, All}`, per node +(`Db::set_envelope_retention`, server `--envelope-retention` / +`ATOMIC_ENVELOPE_RETENTION`, default `latest`). `Latest` keeps the one +envelope that produced the current state, which is the floor (F6). `All` +keeps every envelope, which is the signed audit log (F7). Same write path; +the only difference is whether the prune of older rows runs. + +**Binding to the oplog.** Every commit's Loro change carries a token in +its message: the browser's drain token, and since 2026-09-05 the Rust +builder path (`c-<time>-<rand>`) and `Commit::create_did` (the creator's +subject, as the browser writes it) do the same. History buckets versions +by that token; an envelope's `loroUpdate` names the tokens it introduced; +a version maps to its signer by lookup. A token is credited to the first +retained envelope that carried it (a Rust builder commit ships a full +snapshot), and the genesis carrier token is only ever credited to a +genesis envelope: the genesis is proven by the inline certificate (F1), +not by whoever later shipped a snapshot containing it. + +**Verification.** `envelopes::attribute_history(store, subject)`: +signature check with the same code apply uses, tokens from a probe +import, `complete` = every tokened change in the stored oplog is claimed +by a verified envelope (the genesis carrier excepted, see above; untokened +server bookkeeping such as the `lastCommit` stamp is not counted). Anything +not covered is unattributed, never a guessed signer. + +**Read.** Server `GET /history-attribution?subject=` (read-gated like the +resource; `server/src/handlers/history_attribution.rs`), WASM +`ClientDb.historyAttribution(subject)`, browser +`Store.getHistoryAttribution(subject)` merging both by signature. + +**UI.** History's `VersionTitle` shows `by <agent> Verified` / +`Unverified` from the attribution, and `by peer … Unattributed` when no +envelope covers a version. + +**Tests.** `lib/src/envelopes.rs` (retention, ordering, not-indexed, +attribution, tampering, two writers under both retentions, destroy fold), +`server/tests/it/history_attribution.rs` (signer, verified, read gate), +`browser/lib/src/history-attribution.test.ts` (parse, lookup, merge). + +## Next + +1. **Replicate the rows.** Live `COMMIT` already delivers the envelope; + receivers persist it (they go through `apply_commit`). Bulk: a + capability-gated side map in `SYNC_PUSH` / `SYNC_DIFF`, the shape + `removeCommits` already uses, carrying the retained envelopes of each + pushed subject; the receiver verifies before storing. Vault pack v2 + with an optional per-entry envelope list. A node on `latest` sends one, + a node on `all` sends all. Until this lands a fresh device attributes + only what it applied itself, and the hub answers the rest over + `/history-attribution`. +2. **Secondary indexes** for "everything agent X signed" / "changes in + drive D since T", as a second tree written in the same transaction, + rights-filtered per resource on read. Not before a screen asks. +3. **Session certificates** (#1310): the envelope verify path is the one + place a `sessionCert` chain is checked, with `notAfter` bounds and + fall-back to Unattributed. +4. **Header-only envelopes** as a size win once bodies dominate storage. + +## Resolved open questions + +1. **Container shape.** Side tree, not a Loro container: an envelope + inside the doc would sign a document that contains itself, and a + snapshot import that ignored it would silently drop the log. The side + tree costs one extra field on the wire (item 1 above), which is the + price of proofs that cannot be confused with content. +2. **Who writes.** The node, after verify, in the apply transaction. Two + replicas applying the same `COMMIT` write byte-identical rows under the + same key. +3. **Concurrent edits.** Two envelopes, one merged document: both rows + verified, diffs from Loro. Correct, like two git commits on diverging + branches that later merge. +4. **Size / prune.** `latest` versus `all` per node. `latest` removes + per-change verifiability for older versions, as a shallow clone does; + it is the default because the floor is the current state. +5. **Verify in WASM.** Same signature check as apply; fail closed to + Unattributed, never display a forged signer. ## What not to do -- Local-only blob table that live writes keep and clones never get. +- Local-only blob table that live writes keep and clones never get + (item 1 in *Next* is the fix, not optional). - “Unattributed is fine for new users.” - Re-index envelopes as Atomic resources / `/commits`. - Put the agent DID in the Loro change message and call it proof. - Require a linear `previousCommit` chain. - -## Open questions - -1. **Container shape.** Loro map `envelopes[commitId] = json` vs list of - signed strings. Map is idempotent on retry (same id). -2. **Who writes the container.** Client includes it in `loroUpdate` at - sign time (envelope must then sign a doc that already contains itself - — chicken/egg) **or** server appends after verify (replica that only - has the client delta must apply the same append). Server-append after - verify is simpler; two replicas that both apply the same COMMIT must - append the same bytes so the CRDT converges. -3. **Concurrent edits.** Two envelopes, one merged document: both rows - verified, diffs from Loro. Correct, like two git commits on diverging - branches that later merge. -4. **Size / prune.** Full bodies grow the snapshot. Compaction that drops - old envelopes is a policy on that container, and it *removes* - verifiability for those versions — same as `git replace` / shallow - clone. Default is full log. -5. **Verify in WASM.** Same signature check as apply; fail closed to - Unattributed, never display a forged signer. diff --git a/planning/commit-retention-and-state-certificates.md b/planning/commit-retention-and-state-certificates.md index 50ba57eab..e8c56d65c 100644 --- a/planning/commit-retention-and-state-certificates.md +++ b/planning/commit-retention-and-state-certificates.md @@ -12,10 +12,13 @@ > inherited down the parent chain), capped by node policy — see > "Per-resource retention" below. > -> **Shipped since:** ordinary content commits are discarded after apply; the -> must-retain floor (`AuthImpact::is_critical`) stays. History is Loro-only -> today. Target: a new replica can verify who changed what (`git clone` of -> the log) — [`auditability-loro-history.md`](./auditability-loro-history.md). +> **Shipped since:** ordinary content commits are discarded as resources +> after apply; the must-retain floor (`AuthImpact::is_critical`) stays. The +> signed envelope of every commit is kept per resource in `Tree::Envelopes` +> (`lib/src/envelopes.rs`): the latest one by default, every one under +> `--envelope-retention all`. History reads who signed which change from it +> (`/history-attribution`). Remaining: envelopes travel in bulk sync and the +> vault — [`auditability-loro-history.md`](./auditability-loro-history.md). > > **Current (2026-06-10 server, 2026-07-10 browser).** DID identity is no > longer derived from the genesis *commit* signature. A `did:ad:` subject is diff --git a/planning/completed/commit-retention-floor-decision.md b/planning/completed/commit-retention-floor-decision.md index aa11e79ac..e6d0706ac 100644 --- a/planning/completed/commit-retention-floor-decision.md +++ b/planning/completed/commit-retention-floor-decision.md @@ -1,6 +1,6 @@ # Commit retention floor -**Status:** Accepted 2026-09-01 — option C. Hold #1313 until `Tree::Envelopes` exists; sequence #1274 → #1313 → #1254. +**Status:** Accepted 2026-09-01 — option C. Amended 2026-09-05 (see the end): `Tree::Envelopes` ships inside #1313 itself; #1274 is off the critical path. > **Decision needed by maintainer** > @@ -223,3 +223,33 @@ is retention policy, and the Loro oplog — not commits — is the history.* Unverified: the exact per-commit index-atom count (depends on propvals present); whether the browser OPFS DB stores commit rows via `materializeCommitLocally` (browser side of F6 needs its own check). + + +## Amendment 2026-09-05 + +Built in #1313 (`lib/src/envelopes.rs`); this supersedes the *Minimal +mechanism* and *Sequencing* above where they differ. + +- **Key is the same, retention is a knob.** Key + `pure_id ‖ 0x00 ‖ createdAt ‖ 0x00 ‖ signature`. `EnvelopeRetention` + is `latest` (one row, F6, the default) or `all` (every row, F7). No + `recent N`, no per-class schedule: nothing reads a middle setting. +- **No #1274 gating.** The write sits in `Db::apply_commit`, which every + ingest path already funnels through, in the apply transaction. Ingest + consolidation is orthogonal and lands on its own schedule. +- **Binding to the oplog is explicit.** Every commit's Loro change carries + a token (browser drain token; Rust builder and `create_did` now too). + `attribute_history` maps envelope → tokens → History version, verifies + signatures with the apply code, and reports `complete`. The genesis + carrier token is credited only to a genesis envelope: F1 is the proof + for creation, not whoever later shipped a snapshot. +- **Destroy evidence folds in.** The tombstone value is a marker again; + the destroy envelope is the subject's latest row. +- **Read paths.** `GET /history-attribution`, WASM + `historyAttribution`, `Store.getHistoryAttribution`; History shows + Verified / Unverified / Unattributed. +- **Wire and vault carriage are the next PR**, as a `removeCommits`-style + side map and pack v2 (see `auditability-loro-history.md` → *Next*). + Until then a replica attributes what it applied itself and asks the + hub for the rest. +- **Sequencing now:** #1313 (with envelopes) → #1274 → #1254. diff --git a/server/src/appstate.rs b/server/src/appstate.rs index eebd661f7..3b7511b63 100644 --- a/server/src/appstate.rs +++ b/server/src/appstate.rs @@ -165,6 +165,17 @@ impl AppState { // no request can slip in under the default open policy. crate::host_mode::install_policy(&store, &config.host_mode).await; + match atomic_lib::envelopes::EnvelopeRetention::parse(&config.opts.envelope_retention) { + Some(retention) => store.set_envelope_retention(retention), + None => { + return Err(format!( + "ATOMIC_ENVELOPE_RETENTION must be `latest` or `all`, got `{}`", + config.opts.envelope_retention + ) + .into()) + } + } + let index_status_broadcast = Arc::new(IndexStatusBroadcast::new()); let index_notifier: Arc<dyn Fn(&str, bool) + Send + Sync> = { let b = index_status_broadcast.clone(); diff --git a/server/src/config.rs b/server/src/config.rs index 68670cc96..0b6d2b664 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -31,6 +31,12 @@ pub struct Opts { #[clap(long, env = "ATOMIC_DEVELOPMENT")] pub development: bool, + /// Which signed commit envelopes this node keeps per resource: `latest` + /// (the envelope that produced the current state; the default) or `all` + /// (every envelope, so History shows a verified signer per change). + #[clap(long, default_value = "latest", env = "ATOMIC_ENVELOPE_RETENTION")] + pub envelope_retention: String, + /// The origin domain where the app is hosted, without the port and schema values. #[clap(long, default_value = "localhost", env = "ATOMIC_DOMAIN")] pub domain: String, diff --git a/server/src/handlers/history_attribution.rs b/server/src/handlers/history_attribution.rs new file mode 100644 index 000000000..ebc18d74b --- /dev/null +++ b/server/src/handlers/history_attribution.rs @@ -0,0 +1,46 @@ +//! `GET /history-attribution?subject=<subject>` — who signed a resource's +//! history, as far as this node kept the envelopes (`atomic_lib::envelopes`). +//! +//! Read-gated like the resource itself: the caller must be allowed to read +//! `subject`. The answer is the verified signer per Loro change token, plus +//! whether every client-authored change is covered. A node on `latest` +//! retention answers with one attribution (the current state); `all` gives +//! one per change. + +use crate::{ + appstate::AppState, context::RequestContext, errors::AtomicServerResult, + helpers::get_client_agent, +}; +use actix_web::{web, HttpRequest, HttpResponse}; +use atomic_lib::{Storelike, Subject}; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +pub struct HistoryAttributionParams { + pub subject: String, +} + +#[tracing::instrument(skip_all)] +pub async fn handle_history_attribution( + appstate: web::Data<AppState>, + params: web::Query<HistoryAttributionParams>, + req: HttpRequest, +) -> AtomicServerResult<HttpResponse> { + let store = &appstate.store; + let origin = RequestContext::new(&req, &appstate).origin; + let subject = params.subject.clone(); + + // The client signs the full request URL (path + query); rebuild it exactly + // so the signature check matches what it signed. + let full_url = format!("{}{}", origin, req.uri()); + let for_agent = get_client_agent(req.headers(), &appstate, &full_url).await?; + + // A destroyed subject has no resource to check against; its drive is the + // rights anchor, the same gate `SYNC_DIFF.removeCommits` uses. Only a + // live resource is answered here; destroyed ones are the sync layer's. + let resource = store.get_resource(&Subject::from(subject.as_str())).await?; + atomic_lib::hierarchy::check_read(store, &resource, &for_agent).await?; + + let report = atomic_lib::envelopes::attribute_history(store, &subject).await?; + Ok(HttpResponse::Ok().json(report)) +} diff --git a/server/src/handlers/mod.rs b/server/src/handlers/mod.rs index 59f86e98d..fd20c6a3f 100644 --- a/server/src/handlers/mod.rs +++ b/server/src/handlers/mod.rs @@ -12,6 +12,7 @@ pub mod drive_usage; pub mod export; pub mod forget_peer; pub mod get_resource; +pub mod history_attribution; #[cfg(feature = "image")] pub mod image; pub mod plugin_ui; diff --git a/server/src/routes.rs b/server/src/routes.rs index dcbf04e2d..533eda6f6 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -207,6 +207,10 @@ pub fn config_routes(app: &mut actix_web::web::ServiceConfig) { ) .service(web::resource("/ws").to(handlers::web_sockets::web_socket_handler)) .service(web::resource("/drive-usage").to(handlers::drive_usage::handle_drive_usage)) + .service( + web::resource("/history-attribution") + .to(handlers::history_attribution::handle_history_attribution), + ) .service( web::resource("/forget-peer") .guard(guard::Method(Method::POST)) diff --git a/server/tests/it/history_attribution.rs b/server/tests/it/history_attribution.rs new file mode 100644 index 000000000..72b1ae9bd --- /dev/null +++ b/server/tests/it/history_attribution.rs @@ -0,0 +1,106 @@ +//! `GET /history-attribution?subject=` — the signed-envelope attribution of a +//! resource's history (`atomic_lib::envelopes`), read-gated like the resource. +//! +//! Run: cargo test -p atomic-server --test it history_attribution + +use atomic_lib::{client::connected::Client, errors::AtomicResult}; + +use crate::common::{start_server, wait_for_server}; + +fn url_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{:02X}", b)), + } + } + out +} + +async fn fetch_attribution( + server_url: &str, + subject: &str, + agent: Option<&atomic_lib::agents::Agent>, +) -> AtomicResult<(u16, serde_json::Value)> { + let url = format!( + "{server_url}/history-attribution?subject={}", + url_encode(subject) + ); + let mut req = reqwest::Client::new() + .get(&url) + .header("Accept", "application/json"); + if let Some(agent) = agent { + for (k, v) in atomic_lib::client::get_authentication_headers(&url, agent)? { + req = req.header(k, v); + } + } + let resp = req.send().await.map_err(|e| e.to_string())?; + let status = resp.status().as_u16(); + let body = resp.text().await.unwrap_or_default(); + let json = serde_json::from_str(&body).unwrap_or(serde_json::Value::Null); + Ok((status, json)) +} + +#[tokio::test] +async fn history_attribution_names_the_verified_signer_and_gates_on_read() -> AtomicResult<()> { + let port = start_server("history_attribution"); + wait_for_server(port).await; + let server_url = format!("http://localhost:{port}"); + + let client = Client::new(&server_url).await?; + let alice = client.new_agent("Alice").await?; + // A private drive: only Alice may read what is in it. + let drive = client.new_drive(&alice, "Attribution Drive").await?; + + let mut resource = client.new_resource(&drive)?; + resource.set_name("Attributed")?; + resource.set_unsafe( + atomic_lib::urls::IS_A.into(), + atomic_lib::Value::ResourceArray(vec![atomic_lib::urls::CLASS.into()]), + )?; + resource.set_unsafe( + atomic_lib::urls::SHORTNAME.into(), + atomic_lib::Value::Slug("attributed".into()), + )?; + resource.set_unsafe( + atomic_lib::urls::DESCRIPTION.into(), + atomic_lib::Value::String("signed by Alice".into()), + )?; + let subject = resource.save_remote(client.store()).await?; + + // The signer sees her own envelope, verified by the server. + let (status, report) = fetch_attribution(&server_url, &subject, Some(&alice)).await?; + assert_eq!(status, 200, "owner may read attribution: {report}"); + let attributions = report["attributions"] + .as_array() + .expect("attributions array"); + assert!(!attributions.is_empty(), "the genesis envelope is retained"); + let last = attributions.last().unwrap(); + assert_eq!(last["signer"], alice.subject.to_string()); + assert_eq!(last["verified"], true); + assert_eq!(report["retention"], "latest"); + assert!( + last["tokens"].as_array().is_some_and(|t| !t.is_empty()), + "the envelope names the Loro change it introduced: {last}" + ); + + // A stranger is refused, exactly like the resource itself. + let mallory = client.new_agent("Mallory").await?; + let (status, _) = fetch_attribution(&server_url, &subject, Some(&mallory)).await?; + assert_ne!( + status, 200, + "a stranger must not read a private resource's attribution" + ); + + // So is an anonymous request. + let (status, _) = fetch_attribution(&server_url, &subject, None).await?; + assert_ne!( + status, 200, + "anonymous must not read a private resource's attribution" + ); + + Ok(()) +} diff --git a/server/tests/it/main.rs b/server/tests/it/main.rs index f250815d6..5c3499908 100644 --- a/server/tests/it/main.rs +++ b/server/tests/it/main.rs @@ -10,6 +10,7 @@ mod blob_sync; mod drive_presence; mod drive_presence_shared; mod file_search_repro; +mod history_attribution; mod iroh_pairing; mod loro_ephemeral_sync; mod multi_client_sync; diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 5b7c5bdcf..9bce64ba1 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -388,6 +388,18 @@ impl ClientDb { Self::state_snapshot_js(self.db(), subject) } + /// Who signed this resource's history, from the envelopes this client + /// kept (`atomic_lib::envelopes::attribute_history`). JSON string with + /// `attributions[]` (signer, createdAt, signature, verified, tokens, + /// destroy, genesis), `retention` and `complete`. + #[wasm_bindgen(js_name = "historyAttribution")] + pub async fn history_attribution(&self, subject: &str) -> Result<String, JsError> { + let report = atomic_lib::envelopes::attribute_history(self.db(), subject) + .await + .map_err(to_js_err)?; + serde_json::to_string(&report).map_err(to_js_err) + } + /// Back-compat alias for browser client-db (`getLoroSnapshot`). #[wasm_bindgen(js_name = "getLoroSnapshot")] pub fn get_loro_snapshot(&self, subject: &str) -> Result<JsValue, JsError> { From fcb03db235a92f84fefc64a06a3b2e7951d164ef Mon Sep 17 00:00:00 2001 From: Joep Meindertsma <joep@ontola.io> Date: Sat, 5 Sep 2026 13:29:55 +0200 Subject: [PATCH 7/8] style: oxfmt history-attribution test with the lib config --- browser/lib/src/history-attribution.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/browser/lib/src/history-attribution.test.ts b/browser/lib/src/history-attribution.test.ts index bb2bc63e5..5159d2651 100644 --- a/browser/lib/src/history-attribution.test.ts +++ b/browser/lib/src/history-attribution.test.ts @@ -39,7 +39,13 @@ describe('parseHistoryAttribution', () => { destroy: false, genesis: true, }, - { signer: bob, created_at: 43, signature: 'b', verified: false, tokens: ['c-1'] }, + { + signer: bob, + created_at: 43, + signature: 'b', + verified: false, + tokens: ['c-1'], + }, ], }; const fromObject = parseHistoryAttribution(wire); @@ -92,7 +98,9 @@ describe('attributionForVersion', () => { }); it('leaves untokened or unclaimed versions unattributed', () => { - expect(attributionForVersion({ message: undefined }, report)).toBeUndefined(); + expect( + attributionForVersion({ message: undefined }, report), + ).toBeUndefined(); expect(attributionForVersion({ message: 'c-9' }, report)).toBeUndefined(); expect(attributionForVersion({ message: 'c-7' }, null)).toBeUndefined(); }); From 9484a59ddf535a220fbbc2f744ab706c6f99104c Mon Sep 17 00:00:00 2001 From: Joep Meindertsma <joep@ontola.io> Date: Sat, 5 Sep 2026 14:05:14 +0200 Subject: [PATCH 8/8] fix(history): bind envelopes to deltas by version range, flat i18n-safe VersionTitle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - attribute_history read each envelope's tokens by importing its update into an empty doc; a browser delta has dependencies, sits pending there and lists no changes, so every browser edit came back with no tokens. Tokens are now read from the stored doc over the update's [start, end) range (AtomicLoroDoc::change_messages_in / update_range). Regression test with a real delta. - getLoroHistory strips the drain token from Version.message for display; expose it as Version.token so attributionForVersion can match it. - VersionTitle is split into flat pieces: the i18n extractor turned the element-spanning ternary into one placeholder message that rendered as [i18n-404:…] (the CI failure). Catalogues re-extracted. --- browser/data-browser/src/locales/de.po | 41 ++++++- browser/data-browser/src/locales/en.po | 41 ++++++- browser/data-browser/src/locales/es.po | 41 ++++++- browser/data-browser/src/locales/fr.po | 41 ++++++- .../src/routes/History/VersionTitle.tsx | 77 ++++++++----- browser/lib/src/history-attribution.test.ts | 14 +-- browser/lib/src/history-attribution.ts | 7 +- browser/lib/src/resource.ts | 9 ++ lib/src/envelopes.rs | 105 +++++++++++++----- lib/src/loro.rs | 40 +++++++ 10 files changed, 335 insertions(+), 81 deletions(-) diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index 0cb8b87e5..035732b6a 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -3184,10 +3184,8 @@ msgstr "" msgid "Could not save resource" msgstr "" -#. 0: version.peer && <> by peer {version.peer.slice(0, 8)}...</>; 1: version.message && <> — {version.message}</> -#: src/routes/History/VersionTitle.tsx -msgid "Edited <0/> {0} {1}" -msgstr "" +#~ msgid "Edited <0/> {0} {1}" +#~ msgstr "" #: src/views/OntologyPage/Property/PropertyLineRead.tsx msgid "Property does not exist anymore" @@ -7258,3 +7256,38 @@ msgstr "" #: src/components/Vault/LinkProviderPanel.tsx msgid "stores it without being able to read it." msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Verified" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Unverified" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Signature checked against the signer's key" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Envelope present, but its signature did not verify" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "No signed envelope covers this change on this node" +msgstr "" + +#. 0: ' '; 1: signed ? ( <SignedBy attribution={signed} /> ) \\: ( <UnattributedBy peer={version.peer} /> ); 2: version.message && !signed && <> — {version.message}</> +#: src/routes/History/VersionTitle.tsx +msgid "Edited <0/>{0} {1} {2}" +msgstr "" + +#. 0: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by <0/>{0} <1/>" +msgstr "" + +#. 0: shortPeer; 1: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by peer {0}{1} <0>Unattributed</0>" +msgstr "" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index df07825cc..61ae116c9 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -3184,10 +3184,8 @@ msgstr "Next item" msgid "Could not save resource" msgstr "Could not save resource" -#. 0: version.peer && <> by peer {version.peer.slice(0, 8)}...</>; 1: version.message && <> — {version.message}</> -#: src/routes/History/VersionTitle.tsx -msgid "Edited <0/> {0} {1}" -msgstr "Edited <0/> {0} {1}" +#~ msgid "Edited <0/> {0} {1}" +#~ msgstr "Edited <0/> {0} {1}" #: src/views/OntologyPage/Property/PropertyLineRead.tsx msgid "Property does not exist anymore" @@ -7292,3 +7290,38 @@ msgstr "This app cannot sign in on its own, so approve it from somewhere you alr #: src/components/Vault/LinkProviderPanel.tsx msgid "stores it without being able to read it." msgstr "stores it without being able to read it." + +#: src/routes/History/VersionTitle.tsx +msgid "Verified" +msgstr "Verified" + +#: src/routes/History/VersionTitle.tsx +msgid "Unverified" +msgstr "Unverified" + +#: src/routes/History/VersionTitle.tsx +msgid "Signature checked against the signer's key" +msgstr "Signature checked against the signer's key" + +#: src/routes/History/VersionTitle.tsx +msgid "Envelope present, but its signature did not verify" +msgstr "Envelope present, but its signature did not verify" + +#: src/routes/History/VersionTitle.tsx +msgid "No signed envelope covers this change on this node" +msgstr "No signed envelope covers this change on this node" + +#. 0: ' '; 1: signed ? ( <SignedBy attribution={signed} /> ) \\: ( <UnattributedBy peer={version.peer} /> ); 2: version.message && !signed && <> — {version.message}</> +#: src/routes/History/VersionTitle.tsx +msgid "Edited <0/>{0} {1} {2}" +msgstr "Edited <0/>{0} {1} {2}" + +#. 0: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by <0/>{0} <1/>" +msgstr "by <0/>{0} <1/>" + +#. 0: shortPeer; 1: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by peer {0}{1} <0>Unattributed</0>" +msgstr "by peer {0}{1} <0>Unattributed</0>" diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index a3e0e2953..9bcd402f0 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -3184,10 +3184,8 @@ msgstr "Elemento siguiente" msgid "Could not save resource" msgstr "No se pudo guardar el recurso" -#. 0: version.peer && <> by peer {version.peer.slice(0, 8)}...</>; 1: version.message && <> — {version.message}</> -#: src/routes/History/VersionTitle.tsx -msgid "Edited <0/> {0} {1}" -msgstr "Editado <0/> {0} {1}" +#~ msgid "Edited <0/> {0} {1}" +#~ msgstr "Editado <0/> {0} {1}" #: src/views/OntologyPage/Property/PropertyLineRead.tsx msgid "Property does not exist anymore" @@ -7258,3 +7256,38 @@ msgstr "" #: src/components/Vault/LinkProviderPanel.tsx msgid "stores it without being able to read it." msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Verified" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Unverified" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Signature checked against the signer's key" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Envelope present, but its signature did not verify" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "No signed envelope covers this change on this node" +msgstr "" + +#. 0: ' '; 1: signed ? ( <SignedBy attribution={signed} /> ) \\: ( <UnattributedBy peer={version.peer} /> ); 2: version.message && !signed && <> — {version.message}</> +#: src/routes/History/VersionTitle.tsx +msgid "Edited <0/>{0} {1} {2}" +msgstr "" + +#. 0: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by <0/>{0} <1/>" +msgstr "" + +#. 0: shortPeer; 1: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by peer {0}{1} <0>Unattributed</0>" +msgstr "" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index 5f9251f5c..1a0bcba53 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -3184,10 +3184,8 @@ msgstr "Élément suivant" msgid "Could not save resource" msgstr "Impossible d'enregistrer la ressource" -#. 0: version.peer && <> by peer {version.peer.slice(0, 8)}...</>; 1: version.message && <> — {version.message}</> -#: src/routes/History/VersionTitle.tsx -msgid "Edited <0/> {0} {1}" -msgstr "Modifié <0/> {0} {1}" +#~ msgid "Edited <0/> {0} {1}" +#~ msgstr "Modifié <0/> {0} {1}" #: src/views/OntologyPage/Property/PropertyLineRead.tsx msgid "Property does not exist anymore" @@ -7258,3 +7256,38 @@ msgstr "" #: src/components/Vault/LinkProviderPanel.tsx msgid "stores it without being able to read it." msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Verified" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Unverified" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Signature checked against the signer's key" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "Envelope present, but its signature did not verify" +msgstr "" + +#: src/routes/History/VersionTitle.tsx +msgid "No signed envelope covers this change on this node" +msgstr "" + +#. 0: ' '; 1: signed ? ( <SignedBy attribution={signed} /> ) \\: ( <UnattributedBy peer={version.peer} /> ); 2: version.message && !signed && <> — {version.message}</> +#: src/routes/History/VersionTitle.tsx +msgid "Edited <0/>{0} {1} {2}" +msgstr "" + +#. 0: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by <0/>{0} <1/>" +msgstr "" + +#. 0: shortPeer; 1: ' ' +#: src/routes/History/VersionTitle.tsx +msgid "by peer {0}{1} <0>Unattributed</0>" +msgstr "" diff --git a/browser/data-browser/src/routes/History/VersionTitle.tsx b/browser/data-browser/src/routes/History/VersionTitle.tsx index 87f28a79e..af24bb132 100644 --- a/browser/data-browser/src/routes/History/VersionTitle.tsx +++ b/browser/data-browser/src/routes/History/VersionTitle.tsx @@ -1,5 +1,6 @@ import { attributionForVersion, + type Attribution, type HistoryAttribution, type Version, } from '@tomic/react'; @@ -28,6 +29,11 @@ export interface VersionTitleProps { * Loro change token matches this version, and is labelled Verified when the * answering node checked the signature. A version no envelope claims falls * back to the Loro peer id: that is who typed, not a proof of who signed. + * + * Kept as small, flat pieces of JSX text: the i18n extractor (wuchale) turns + * each text run into a catalogue entry, and a ternary spanning elements + * extracts as one placeholder-heavy message that renders as `[i18n-404:…]` + * until translated. */ export function VersionTitle({ version, @@ -39,43 +45,56 @@ export function VersionTitle({ return ( <span> - Edited <time dateTime={date.toISOString()}>{formattedDate}</time> + Edited <time dateTime={date.toISOString()}>{formattedDate}</time>{' '} {signed ? ( - <> - {' by '} - <ResourceInline subject={signed.signer} />{' '} - <Badge - $verified={signed.verified} - title={ - signed.verified - ? "Signature checked against the signer's key" - : 'Envelope present, but its signature did not verify' - } - data-testid='version-attribution' - > - {signed.verified ? 'Verified' : 'Unverified'} - </Badge> - </> + <SignedBy attribution={signed} /> ) : ( - version.peer && ( - <> - {' by peer '} - {version.peer.slice(0, 8)}...{' '} - <Badge - $verified={false} - title='No signed envelope covers this change on this node' - data-testid='version-attribution' - > - Unattributed - </Badge> - </> - ) + <UnattributedBy peer={version.peer} /> )} {version.message && !signed && <> — {version.message}</>} </span> ); } +function SignedBy({ attribution }: { attribution: Attribution }): JSX.Element { + const label = attribution.verified ? 'Verified' : 'Unverified'; + const title = attribution.verified + ? "Signature checked against the signer's key" + : 'Envelope present, but its signature did not verify'; + + return ( + <> + by <ResourceInline subject={attribution.signer} />{' '} + <Badge + $verified={attribution.verified} + title={title} + data-testid='version-attribution' + > + {label} + </Badge> + </> + ); +} + +function UnattributedBy({ peer }: { peer?: string }): JSX.Element | null { + if (!peer) return null; + + const shortPeer = `${peer.slice(0, 8)}...`; + + return ( + <> + by peer {shortPeer}{' '} + <Badge + $verified={false} + title='No signed envelope covers this change on this node' + data-testid='version-attribution' + > + Unattributed + </Badge> + </> + ); +} + const Badge = styled.span<{ $verified: boolean }>` display: inline-block; padding: 0 0.4em; diff --git a/browser/lib/src/history-attribution.test.ts b/browser/lib/src/history-attribution.test.ts index 5159d2651..078271411 100644 --- a/browser/lib/src/history-attribution.test.ts +++ b/browser/lib/src/history-attribution.test.ts @@ -91,18 +91,16 @@ describe('attributionForVersion', () => { }; it('maps a version to the envelope carrying its token', () => { + expect(attributionForVersion({ token: 'c-7' }, report)?.signer).toBe(bob); + expect(attributionForVersion({ token: alice }, report)?.genesis).toBe(true); + // `message` is the display field; a legacy caller may still pass it. expect(attributionForVersion({ message: 'c-7' }, report)?.signer).toBe(bob); - expect(attributionForVersion({ message: alice }, report)?.genesis).toBe( - true, - ); }); it('leaves untokened or unclaimed versions unattributed', () => { - expect( - attributionForVersion({ message: undefined }, report), - ).toBeUndefined(); - expect(attributionForVersion({ message: 'c-9' }, report)).toBeUndefined(); - expect(attributionForVersion({ message: 'c-7' }, null)).toBeUndefined(); + expect(attributionForVersion({ token: undefined }, report)).toBeUndefined(); + expect(attributionForVersion({ token: 'c-9' }, report)).toBeUndefined(); + expect(attributionForVersion({ token: 'c-7' }, null)).toBeUndefined(); }); }); diff --git a/browser/lib/src/history-attribution.ts b/browser/lib/src/history-attribution.ts index 726567f16..15c313307 100644 --- a/browser/lib/src/history-attribution.ts +++ b/browser/lib/src/history-attribution.ts @@ -100,11 +100,12 @@ export function parseHistoryAttribution( * carries is unattributed. */ export function attributionForVersion( - version: Pick<Version, 'message'>, + version: Pick<Version, 'token' | 'message'>, report: HistoryAttribution | null | undefined, ): Attribution | undefined { - if (!report || !version.message) return undefined; - const token = version.message; + const token = version.token ?? version.message; + + if (!report || !token) return undefined; return report.attributions.find(a => a.tokens.includes(token)); } diff --git a/browser/lib/src/resource.ts b/browser/lib/src/resource.ts index 42199d248..8d3bb60a4 100644 --- a/browser/lib/src/resource.ts +++ b/browser/lib/src/resource.ts @@ -2135,6 +2135,9 @@ export class Resource<C extends OptionalClass = any> { // (`c-<ulid>`), not a human-authored message — don't surface it in the // history UI. Left undefined until real commit messages exist. message: undefined, + // The raw token, for matching this version to the signed envelope + // that introduced it (`attributionForVersion`). + token: g.step.message, propvals: g.propvals, containers: g.containers, })); @@ -3786,6 +3789,12 @@ export interface Version { frontiers: any[]; /** Human-readable commit message, if set */ message?: string; + /** + * The Loro change message this version was bucketed by: the drain's + * `c-<ulid>` commit token or, at genesis, the creator's agent subject. + * Not for display; it is what a signed envelope's `tokens` name. + */ + token?: string; /** Materialized property values at this version */ propvals: Map<string, JSONValue>; /** Top-level Loro containers besides `properties` at this version — most diff --git a/lib/src/envelopes.rs b/lib/src/envelopes.rs index f8d67bcb0..24774368d 100644 --- a/lib/src/envelopes.rs +++ b/lib/src/envelopes.rs @@ -237,6 +237,16 @@ pub struct HistoryAttribution { pub async fn attribute_history(store: &Db, subject: &str) -> AtomicResult<HistoryAttribution> { let retention = store.envelope_retention().as_str(); let mut attributions: Vec<Attribution> = Vec::new(); + let pure = crate::Subject::from_raw(subject, None).pure_id(); + // The stored doc is where an update's changes can be read back with + // their messages: a delta carries only its own ops, and only a doc that + // already holds their dependencies can list them. + let stored_doc = store + .kv + .get(Tree::LoroSnapshots, pure.as_bytes()) + .ok() + .flatten() + .and_then(|bytes| crate::loro::AtomicLoroDoc::from_snapshot(&bytes).ok()); for envelope in envelopes(store, subject) { let resource = crate::parse::parse_json_ad_commit_resource(&envelope.json, store).await?; @@ -253,18 +263,15 @@ pub async fn attribute_history(store: &Db, subject: &str) -> AtomicResult<Histor // it: only a genesis envelope may claim it. let is_genesis = commit.is_genesis == Some(true); let mut tokens = Vec::new(); - if let Some(update) = commit.loro_update.as_deref() { - let probe = crate::loro::AtomicLoroDoc::new(); - if probe.import_update(update).is_ok() { - for change in probe.get_history() { - if let Some(message) = change.message { - if is_genesis_carrier(&message) && !is_genesis { - continue; - } - let claimed = attributions.iter().any(|a| a.tokens.contains(&message)); - if !claimed && !tokens.contains(&message) { - tokens.push(message); - } + if let (Some(update), Some(doc)) = (commit.loro_update.as_deref(), stored_doc.as_ref()) { + if let Ok((start, end)) = crate::loro::AtomicLoroDoc::update_range(update) { + for message in doc.change_messages_in(&start, &end) { + if is_genesis_carrier(&message) && !is_genesis { + continue; + } + let claimed = attributions.iter().any(|a| a.tokens.contains(&message)); + if !claimed && !tokens.contains(&message) { + tokens.push(message); } } } @@ -281,19 +288,12 @@ pub async fn attribute_history(store: &Db, subject: &str) -> AtomicResult<Histor }); } - let pure = crate::Subject::from_raw(subject, None).pure_id(); - let stored_tokens: Option<Vec<String>> = store - .kv - .get(Tree::LoroSnapshots, pure.as_bytes()) - .ok() - .flatten() - .and_then(|bytes| crate::loro::AtomicLoroDoc::from_snapshot(&bytes).ok()) - .map(|doc| { - doc.get_history() - .into_iter() - .filter_map(|change| change.message) - .collect() - }); + let stored_tokens: Option<Vec<String>> = stored_doc.as_ref().map(|doc| { + doc.get_history() + .into_iter() + .filter_map(|change| change.message) + .collect() + }); // The genesis change is covered by the resource's inline certificate // (F1), so it is not required here; every other tokened change must be. let complete = match stored_tokens { @@ -580,6 +580,61 @@ mod tests { } } + /// The browser signs a *delta* (ops since its last save) tagged with a + /// drain token, not a snapshot. Its tokens must still be read: a delta + /// imported into an empty doc is pending (missing deps) and lists no + /// changes, which is how attribution first shipped with empty tokens. + #[tokio::test] + async fn browser_style_delta_envelope_is_attributed_by_its_token() { + let db = Db::init_temp("envelopes_delta").await.unwrap(); + db.set_envelope_retention(EnvelopeRetention::All); + let (alice, drive) = db.setup("Alice").await.unwrap(); + let subject = child(&db, &drive).await; + + // A client that holds the current state edits it and exports only + // the new change, tagged the way the drain tags it. + let resource = db.get_resource(&subject).await.unwrap(); + let snapshot = resource.materialized_state().expect("stored state"); + let base_vv = crate::loro::AtomicLoroDoc::from_snapshot(&snapshot) + .unwrap() + .oplog_vv(); + let client_doc = crate::loro::AtomicLoroDoc::from_snapshot(&snapshot).unwrap(); + client_doc + .set_property(urls::NAME, &Value::String("delta edit".into())) + .unwrap(); + client_doc.commit_with_message("c-01test-delta-token"); + let delta = client_doc.export_updates_since(&base_vv); + assert!(!delta.is_empty()); + + let mut builder = crate::commit::CommitBuilder::new(subject.clone()); + builder.set_loro_update(delta); + let commit = builder.sign(&alice, &db, &resource).await.unwrap(); + let json = commit + .into_resource(&db) + .await + .unwrap() + .to_json_ad(None) + .unwrap(); + ingest_commit_json(&db, &json, &CommitIngestOpts::peer()) + .await + .expect("the delta applies"); + + let report = attribute_history(&db, subject.as_str()).await.unwrap(); + let last = report.attributions.last().unwrap(); + assert!(last.verified); + assert_eq!( + last.tokens, + vec!["c-01test-delta-token".to_string()], + "the delta's own change token is credited to its envelope" + ); + assert!(report.complete, "{report:?}"); + let stored = db.get_resource(&subject).await.unwrap(); + assert!(crate::history::versions(&stored) + .unwrap() + .iter() + .any(|v| v.message.as_deref() == Some("c-01test-delta-token"))); + } + #[tokio::test] async fn destroy_envelope_is_the_latest_row_of_a_destroyed_subject() { let db = Db::init_temp("envelopes_destroy").await.unwrap(); diff --git a/lib/src/loro.rs b/lib/src/loro.rs index 3e905aee2..56b01cc44 100644 --- a/lib/src/loro.rs +++ b/lib/src/loro.rs @@ -345,6 +345,46 @@ impl AtomicLoroDoc { .collect()) } + /// Messages of the changes this doc holds inside `[start, end)` per peer, + /// i.e. the changes an update with that blob range introduced. Reading + /// them here, from a doc that already has the update's dependencies, + /// is what makes it work for a delta: importing a delta into an empty + /// doc leaves it pending (missing deps) and shows no changes at all. + pub fn change_messages_in(&self, start: &VersionVector, end: &VersionVector) -> Vec<String> { + let mut messages = Vec::new(); + let frontier_ids: Vec<loro::ID> = self.doc.oplog_frontiers().iter().collect(); + if frontier_ids.is_empty() { + return messages; + } + let _ = self + .doc + .travel_change_ancestors(&frontier_ids, &mut |change| { + let peer = change.id.peer; + let from = start.get(&peer).copied().unwrap_or(0); + let to = end.get(&peer).copied().unwrap_or(0); + let change_end = change.id.counter + change.len as i32; + // Overlap with [from, to): a change is one commit boundary, + // so any overlap means this update carried it. + if change.id.counter < to && change_end > from { + if let Some(message) = change.message.as_ref() { + let message = message.to_string(); + if !messages.contains(&message) { + messages.push(message); + } + } + } + ControlFlow::Continue(()) + }); + messages + } + + /// The `[start, end)` version range an update or snapshot blob covers. + pub fn update_range(update: &[u8]) -> AtomicResult<(VersionVector, VersionVector)> { + let meta = LoroDoc::decode_import_blob_meta(update, false) + .map_err(|e| format!("Failed to decode Loro blob meta: {e}"))?; + Ok((meta.partial_start_vv, meta.partial_end_vv)) + } + /// Get the raw oplog version vector. pub fn oplog_vv(&self) -> VersionVector { self.doc.oplog_vv()