diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c42..b617549f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -21,24 +21,24 @@ use crate::webhooks; /// The git all-zeros object id — the create/delete sentinel in a ref update. const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; -/// The set of blob OIDs withheld from **anonymous** replication for a repo, or -/// `None` when the repo must not replicate at all (private / mode A / -/// undetermined — fail closed). This is the anonymous replication gate: -/// `caller` is hard-coded to `None` and there is intentionally no caller -/// parameter, which distinguishes it from the per-caller read-serve projection -/// in `git_upload_pack` (which passes the real caller). Both the push pin path -/// and the reconciliation sweep call this helper so the two cannot drift on -/// what is withheld. `rules` is the already-fetched visibility-rule snapshot +/// The set of blob and withheld-subtree tree OIDs withheld from **anonymous** +/// replication for a repo, or `None` when the repo must not replicate at all +/// (private / mode A / undetermined, fail closed). This is the anonymous +/// replication gate: `caller` is hard-coded to `None` and there is intentionally +/// no caller parameter, which distinguishes it from the per-caller read-serve +/// projection in `git_upload_pack` (which passes the real caller). Both the push +/// pin path and the reconciliation sweep call this helper so the two cannot drift +/// on what is withheld. `rules` is the already-fetched visibility-rule snapshot /// (callers fetch once and may reuse it, e.g. for encrypt-then-pin). /// /// Returns `(announce, withheld)`: `announce` is whether the repo may be /// announced/replicated to the anonymous public at all (also gates gossip and -/// Arweave anchoring downstream), and `withheld` is the anonymous withheld blob -/// set when announceable (`None` when not announceable). A failed/panicked -/// withheld walk fails closed on both axes: `announce` is forced false and -/// `withheld` is `None`, so an unvetted push neither replicates blobs nor -/// announces. Returning both keeps the gate's announce decision a single -/// source rather than recomputing it at each call site. +/// Arweave anchoring downstream), and `withheld` is the anonymous withheld object +/// set (blobs and withheld-subtree trees) when announceable (`None` when not +/// announceable). A failed/panicked withheld walk fails closed on both axes: +/// `announce` is forced false and `withheld` is `None`, so an unvetted push +/// neither replicates objects nor announces. Returning both keeps the gate's +/// announce decision a single source rather than recomputing it at each call site. /// /// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): /// by the time the receive-pack tail calls this, the handler's write permit has @@ -64,13 +64,13 @@ async fn replication_withheld_set( } let withheld = match rules { // No path-scoped rule can withhold anything (covers the empty-rules and - // root-only-rules cases), so skip the full withheld_blob_oids walk and + // root-only-rules cases), so skip the full withheld_object_oids walk and // withhold nothing. The predicate's safety-invariant test guards that // this short-circuit matches what the walk would have returned. Some(rules) if !visibility_pack::has_path_scoped_rule(&rules) => { Some(std::collections::HashSet::new()) } - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep + // withheld_object_oids walks every ref with blocking `git ls-tree`; keep // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); @@ -83,18 +83,18 @@ async fn replication_withheld_set( // The permit lives inside the blocking closure: a started walk // always completes holding it. let _permit = permit; - crate::git::visibility_pack::withheld_blob_oids_bounded( + crate::git::visibility_pack::withheld_object_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await .map_err(|e| { - tracing::warn!(err = %e, "withheld_blob_oids task panicked; skipping replication") + tracing::warn!(err = %e, "withheld_object_oids task panicked; skipping replication") }) .ok() .and_then(|r| { r.map_err(|e| { - tracing::warn!(err = %e, "withheld_blob_oids failed; skipping replication") + tracing::warn!(err = %e, "withheld_object_oids failed; skipping replication") }) .ok() }) @@ -111,16 +111,17 @@ async fn replication_withheld_set( } } -/// The replicable object set for a full-scan pin fallback, failing closed (#99). +/// The replicable object set for a full-scan pin fallback, failing closed (#99, #172). /// The full-scan candidate set includes dangling objects the reachable-only /// withheld set never classified, so compute the reachable visibility-allowed -/// blob set and the all-blob universe off the async worker and keep only -/// non-blobs plus allowed blobs. Any error in either walk (or a task panic) +/// blob set, the reachable visibility-allowed tree set, and the all-blob plus +/// all-tree universes off the async worker and keep only commits/tags plus +/// allowed blobs plus allowed trees. Any error in any walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. /// /// Always walks (there is no no-git arm), so the whole blocking scan runs under -/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// one `git_encrypt_semaphore` admission permit (#174 F4): see /// `acquire_scan_permit` for the defer rationale and the honest residuals. #[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( @@ -139,15 +140,16 @@ async fn fail_closed_full_scan_objects( crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; - // One whole-scan deadline shared across both phases (#174 F4). A fresh - // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus - // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the - // deadline caps total occupancy at ~1x: phase 1 runs against the remaining - // budget, and if it consumes the budget phase 2 gets what is left and fails - // closed (pins nothing) rather than over-holding — the safe direction. The cost - // is honest: a genuinely large repo whose phase 1 nears the budget under-pins - // this push rather than the previous silent ~2x hold; size the budget so both - // phases normally fit. + // One whole-scan deadline shared across every phase (#174 F4). A fresh + // `Instant::now() + timeout` for a later phase let a large-but-successful + // earlier phase plus a full later phase hold the scan permit ~Nx the + // configured budget. Sharing the deadline caps total occupancy at ~1x: + // each phase runs against the remaining budget, and if an earlier phase + // consumes the budget a later one gets what is left and fails closed + // (pins nothing) rather than over-holding, the safe direction. The cost + // is honest: a genuinely large repo whose first phase nears the budget + // under-pins this push rather than the previous silent ~2x hold; size the + // budget so the blob walk, tree walk, and typed enumeration normally fit. let deadline = std::time::Instant::now() + timeout; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, @@ -157,19 +159,32 @@ async fn fail_closed_full_scan_objects( is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; + let allowed_trees = crate::git::visibility_pack::replicable_tree_set_bounded( + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, + )?; + let (all_blobs, all_trees) = + crate::git::push_delta::all_blob_and_tree_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( - candidates, &allowed, &all_blobs, + candidates, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, )) }) .await .map_err(|e| { - tracing::warn!(err = %e, "fail-closed blob walk task panicked; pinning nothing this push") + tracing::warn!(err = %e, "fail-closed object walk task panicked; pinning nothing this push") }) .ok() .and_then(|r| { r.map_err(|e| { - tracing::warn!(err = %e, "fail-closed blob walk failed; pinning nothing this push") + tracing::warn!(err = %e, "fail-closed object walk failed; pinning nothing this push") }) .ok() }) @@ -2444,10 +2459,10 @@ async fn post_receive_replication_tail( did: String, ) { // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means this push pins nothing (private / mode A / - // undetermined, or a walk that failed): skip every pin so even commit and tree - // objects (which withheld_blob_oids never lists) stay local. Fail closed: a + // may read this repo at all and, if so, which blob and withheld-subtree tree + // OIDs must not leave the node. `withheld == None` means this push pins nothing + // (private / mode A / undetermined, or a walk that failed): skip every pin so + // even commit and tag objects (which the withheld walk never lists) stay local. Fail closed: a // private or undetermined repo never leaks. The announce decision that gates // the network-facing sends is taken separately, below. let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); @@ -3883,6 +3898,229 @@ mod tests { ); } + /// #172: first-push delta (empty old tips, `full_scan == false`) must drop the + /// withheld-subtree tree. The withheld set comes from `replication_withheld_set`, + /// the production wrapper all three delta call sites already use. + #[tokio::test] + async fn first_push_delta_drops_withheld_subtree_tree() { + use std::process::Command; + use std::time::Duration; + + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &std::path::Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let head = oid("HEAD"); + let root_tree = oid("HEAD^{tree}"); + let public_tree = oid("HEAD:public"); + let public_blob = oid("HEAD:public/a.txt"); + let secret_tree = oid("HEAD:secret"); + let secret_blob = oid("HEAD:secret/b.txt"); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(64)); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + bare.clone(), + vec![head.clone()], + vec![], + "git".into(), + Duration::from_secs(600), + false, + ) + .await; + assert!( + !pin_set.full_scan, + "a first push of a commit tip must take the delta path, not full-scan" + ); + assert!( + pin_set.candidates.contains(&secret_tree) && pin_set.candidates.contains(&secret_blob), + "precondition: the first-push delta includes the withheld tree and blob, \ + so a later drop is a filter hit, not a missing-candidate skip" + ); + + let (_announce, withheld) = replication_withheld_set( + sem, + Some(vec![vis_rule("/secret/**", &[])]), + OWNER_DID, + true, + bare, + "git".into(), + Duration::from_secs(600), + ) + .await; + let withheld = withheld + .expect("announceable public repo with a path-scoped rule must yield a withheld set"); + let replicable = + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld); + assert!( + replicable.contains(&head) && replicable.contains(&root_tree), + "commit and root tree still replicate" + ); + assert!( + replicable.contains(&public_tree) && replicable.contains(&public_blob), + "public tree and blob still replicate" + ); + assert!( + !replicable.contains(&secret_tree), + "withheld-subtree tree must not survive the first-push delta pin set (#172)" + ); + assert!( + !replicable.contains(&secret_blob), + "withheld blob still drops" + ); + } + + /// #172: full-scan fallback drops a withheld-subtree tree and a dangling tree, + /// keeping the commit, root tree, public tree, and public blob. + #[tokio::test] + async fn full_scan_drops_withheld_and_dangling_trees() { + use std::io::Write; + use std::process::Command; + use std::time::Duration; + + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &std::path::Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let head = oid("HEAD"); + let root_tree = oid("HEAD^{tree}"); + let public_tree = oid("HEAD:public"); + let public_blob = oid("HEAD:public/a.txt"); + let secret_tree = oid("HEAD:secret"); + let secret_blob = oid("HEAD:secret/b.txt"); + + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_blob}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling_tree = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let deadline = std::time::Instant::now() + Duration::from_secs(600); + let candidates = crate::git::push_delta::list_all_objects(&bare, "git", deadline).unwrap(); + assert!( + candidates.contains(&dangling_tree), + "precondition: dangling tree is in the full-scan universe" + ); + assert!( + candidates.contains(&secret_tree) && candidates.contains(&secret_blob), + "precondition: full-scan universe includes the withheld tree and blob" + ); + + let replicable = fail_closed_full_scan_objects( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + bare, + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + "git".into(), + Duration::from_secs(600), + ) + .await; + assert!(replicable.contains(&head), "HEAD commit kept"); + assert!(replicable.contains(&root_tree), "root tree kept"); + assert!(replicable.contains(&public_tree), "public tree kept"); + assert!(replicable.contains(&public_blob), "public blob kept"); + assert!( + !replicable.contains(&secret_tree), + "withheld-subtree tree dropped on full-scan (#172)" + ); + assert!( + !replicable.contains(&secret_blob), + "withheld blob dropped on full-scan" + ); + assert!( + !replicable.contains(&dangling_tree), + "dangling tree dropped on full-scan (#172, #99 geometry for trees)" + ); + } + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + @@ -6859,7 +7097,7 @@ mod tests { // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget // would let it finish. - let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) case \"$*\" in *-rzt*) : ;; *) sleep 1.5 ;; esac ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; let git_bin = write_fake_git(tmp.path(), body); // A candidate that is NOT a blob (never appears in all_blob_oids): kept by // replicable_objects_fail_closed only if phase 2 actually ran to completion. @@ -6885,6 +7123,45 @@ mod tests { ); } + /// #172 / #174 F4 sibling: a successful blob walk that nearly exhausts the + /// shared budget must leave the tree allowed-set walk only the remainder, so + /// the scan pins nothing. Distinguishes `ls-tree -rzt` (tree) from `ls-tree -rz` + /// (blob). Load-bearing together with a fresh timeout on both the tree walk and + /// the typed enumeration: those two later phases completing independently would + /// keep the non-blob non-tree candidate. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_with_tree_allowed_set_walk() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Blob ls-tree -rz sleeps 1.5s; tree ls-tree -rzt sleeps 1.5s; cat-file is + // instant. With a 2s whole-scan budget the tree walk is reaped. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) case \"$*\" in *-rzt*) sleep 1.5 ;; *) sleep 1.5 ;; esac ;;\n cat-file) : ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful blob walk must leave the tree allowed-set walk only \ + the SHARED remainder, so it reaps and the scan fails closed; got {objs:?}" + ); + } + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full /// object-id list while walking it, so concurrent pin loops across many repos must /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 0b569693..e13b65af 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -19,7 +19,7 @@ //! objects the reachable withheld set never classified, so subtracting that set //! is not enough — a dangling private blob would slip through (#99). The caller //! signals a full scan via [`PinCandidateSet::full_scan`] and must then apply -//! the fail-closed blob filter (`visibility_pack::replicable_objects_fail_closed`) +//! the fail-closed blob-and-tree filter (`visibility_pack::replicable_objects_fail_closed`) //! instead of the plain reachable-only subtraction. //! //! Because the pin candidate set needs only the OID *set* (never the per-path @@ -211,8 +211,9 @@ pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> R /// Like [`list_all_objects`] but pairs each OID with its object type, via /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed -/// filter needs to tell blobs (content, withholdable) from commits/trees -/// (structural, never withheld) without typing the candidate list itself. +/// filter needs to tell blobs (content, withholdable) and trees (content, +/// withholdable since #135/#173) from commits/tags (structural, never withheld) +/// without typing the candidate list itself. pub fn list_all_objects_with_type( repo_path: &Path, git_bin: &str, @@ -244,24 +245,48 @@ pub fn list_all_objects_with_type( /// Every blob OID in the repo, including unreachable/dangling ones. The /// fail-closed pin filter drops any candidate blob absent from the reachable, /// visibility-allowed set; a dangling private blob is in this set but not the -/// allowed set, so it never replicates (#99). +/// allowed set, so it never replicates (#99). Production uses +/// [`all_blob_and_tree_oids`] (one typed pass); this wrapper stays for tests. +#[cfg(test)] pub fn all_blob_oids( repo_path: &Path, git_bin: &str, deadline: Instant, ) -> Result> { - Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? - .into_iter() - .filter(|(_, ty)| ty == "blob") - .map(|(oid, _)| oid) - .collect()) + Ok(all_blob_and_tree_oids(repo_path, git_bin, deadline)?.0) +} + +/// Every blob OID and every tree OID in the repo, including unreachable/dangling +/// ones, from one typed enumeration. The fail-closed pin filter drops any +/// candidate blob or tree absent from the matching reachable visibility-allowed +/// set; a dangling or withheld object is in this universe but not the allowed +/// set, so it never replicates (#99, #172). +pub fn all_blob_and_tree_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result<(HashSet, HashSet)> { + let mut blobs = HashSet::new(); + let mut trees = HashSet::new(); + for (oid, ty) in list_all_objects_with_type(repo_path, git_bin, deadline)? { + match ty.as_str() { + "blob" => { + blobs.insert(oid); + } + "tree" => { + trees.insert(oid); + } + _ => {} + } + } + Ok((blobs, trees)) } /// The pin candidate OIDs for a push plus whether they came from a full-repo /// scan. `full_scan` is true when the delta could not be used and the whole /// object DB (including dangling objects) was enumerated — the caller must then -/// apply the fail-closed blob filter, because the reachable-only withheld set -/// cannot classify dangling blobs (#99). +/// apply the fail-closed blob-and-tree filter, because the reachable-only withheld set +/// cannot classify dangling blobs or trees (#99, #172). #[derive(Debug, Clone, PartialEq, Eq)] pub struct PinCandidateSet { pub candidates: Vec, diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 08666994..3a5373a8 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -1,7 +1,9 @@ -//! Resolve which blob OIDs must be withheld from a caller because every path -//! at which the blob appears is denied by the repo's visibility rules. Trees -//! and commits are never withheld (mode B keeps SHAs intact); only blob -//! content is held back. +//! Resolve which object OIDs must be withheld from a caller because every path +//! at which the object appears is denied by the repo's visibility rules. Blobs +//! and subtree trees are withholdable (mode B; trees since #135/#173). Commits +//! and tags are never withheld: they are root-level metadata a "/" reader already +//! clears. The replication pin path uses [`withheld_object_oids_bounded`] so a +//! withheld-subtree tree is not exported to public IPFS (#172). use crate::db::VisibilityRule; use crate::visibility::{visibility_check, Decision}; @@ -540,6 +542,79 @@ pub fn withheld_blob_oids_bounded( )) } +/// Reachable blob AND withheld-subtree tree OIDs the anonymous (or `caller`) +/// replication audience must not receive. The single source for +/// `replication_withheld_set`: a blob-only set handed to [`replicable_objects`] +/// reintroduces #172 (withheld trees pin to public IPFS). +/// +/// Strict where [`tree_paths`] is lenient. This feeds a withheld filter, so +/// absence means replicate; a missed reachable tree would leak. The walk +/// therefore matches [`blob_paths`]: [`assert_all_refs_are_commits`] first, then +/// a failing `rev-list --all [HEAD]` is an error, then [`object_paths`] (already +/// fail-closed on ls-tree / non-UTF-8) plus [`root_tree_pairs`]. [`tree_paths`] +/// stays the lenient allow-list walk for GET /ipfs/{cid} and the full-scan +/// allowed-tree set, where absence withholds. +pub fn withheld_object_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits: Vec = String::from_utf8_lossy(&commits_out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + let triples = object_paths(repo_path, git_bin, &commits, deadline)?; + let mut pairs: Vec<(String, String)> = triples + .into_iter() + .filter(|(_, _, kind)| kind == "blob" || kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + pairs.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(withheld_from_pairs( + &pairs, rules, is_public, owner_did, caller, + )) +} + +/// [`withheld_object_oids_bounded`] with the test walk budget. +#[cfg(test)] +pub fn withheld_object_oids( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + withheld_object_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + /// Withheld set from an already-computed (oid, "/path") listing: a blob is /// withheld only when visibility denies the caller at *every* path it appears /// at. Split out so a caller that already walked `blob_paths` (e.g. @@ -588,8 +663,9 @@ pub fn has_path_scoped_rule(rules: &[VisibilityRule]) -> bool { /// Objects that may replicate to the public: everything not in `withheld`. /// Order-preserving. The single seam every replication site (IPFS, Pinata) -/// passes its object list through; option B would later reroute the withheld -/// ones through encrypt-then-pin instead of dropping them. +/// passes its object list through on the delta path. `withheld` MUST be the +/// combined blob-and-tree set from [`withheld_object_oids_bounded`] (via +/// `replication_withheld_set`); a blob-only set reintroduces #172. pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec { all.into_iter() .filter(|oid| !withheld.contains(oid)) @@ -627,6 +703,23 @@ pub fn replicable_blob_set_bounded( ) } +/// Reachable tree OIDs that visibility ALLOWS the anonymous replication +/// audience at some path. The tree analog of [`replicable_blob_set_bounded`], +/// used on the fail-closed full-scan pin path so a withheld or dangling tree +/// is absent and dropped (#172). +pub fn replicable_tree_set_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -827,11 +920,12 @@ fn root_tree_pairs( /// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` /// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every /// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the -/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree -/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed -/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two -/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], -/// bounded by the same shared `deadline`. +/// reachable-commit set ONCE (leniently, see [`reachable_commit_oids`]; the tree +/// allowed-set feeds the `/ipfs/{cid}` tree gate AND the full-scan pin allow-list, +/// where absence = fail-closed) and drives both the ls-tree walk and the root-tree +/// pass from it, so the two cannot diverge and neither re-enumerates. The tree analog +/// of [`blob_paths`], bounded by the same shared `deadline`. Not for a withheld +/// filter: that path is [`withheld_object_oids_bounded`]. fn tree_paths( repo_path: &Path, git_bin: &str, @@ -1150,22 +1244,29 @@ pub fn reachable_commit_tag_oids_bounded( Ok(set) } -/// Objects safe to replicate, failing closed on blobs (#99). A candidate -/// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are -/// structural, never content-withheld) OR it is in `allowed_blobs` (reachable -/// and visibility-allowed). This drops both withheld reachable blobs and -/// dangling/unreachable blobs the reachable walk never classified, without -/// tagging the candidate list with per-object types. Used on the full-scan pin -/// path, where the candidate set can contain dangling objects the reachable-only -/// withheld set cannot cover; the delta path keeps `replicable_objects`. +/// Objects safe to replicate, failing closed on blobs AND trees (#99, #172). +/// A candidate replicates iff it is neither a known blob nor a known tree +/// (commits and tags: structural, a "/" reader already clears) OR it is in +/// `allowed_blobs` OR it is in `allowed_trees`. Trees have been content-withholdable +/// since #135/#173; the old premise that they always pass is falsified. A dangling +/// or withheld tree is in `all_tree_oids` and absent from `allowed_trees`, so it +/// drops. Used on the full-scan pin path, where the candidate set can contain +/// dangling objects the reachable-only withheld set cannot cover; the delta path +/// keeps [`replicable_objects`]. pub fn replicable_objects_fail_closed( candidates: Vec, allowed_blobs: &HashSet, all_blob_oids: &HashSet, + allowed_trees: &HashSet, + all_tree_oids: &HashSet, ) -> Vec { candidates .into_iter() - .filter(|oid| !all_blob_oids.contains(oid) || allowed_blobs.contains(oid)) + .filter(|oid| { + let is_blob = all_blob_oids.contains(oid); + let is_tree = all_tree_oids.contains(oid); + (!is_blob && !is_tree) || allowed_blobs.contains(oid) || allowed_trees.contains(oid) + }) .collect() } @@ -2384,28 +2485,49 @@ esac\n"; } #[test] - fn fail_closed_keeps_nonblobs_and_allowed_blobs_only() { - // Non-blob objects (commit/tree) always pass; a blob passes only if it - // is in the allowed set. A withheld blob and a dangling blob (both in - // all_blob_oids, neither in allowed) are dropped. - let allowed: HashSet = ["b_pub".to_string()].into_iter().collect(); + fn fail_closed_keeps_commits_allowed_blobs_and_allowed_trees_only() { + // Deliberate invariant flip of fail_closed_keeps_nonblobs_and_allowed_blobs_only + // (#172). That test's premise ("commits and trees are structural, never + // content-withheld") was falsified by #135/#173: a withheld-subtree tree is + // content, same as a withheld blob. Commits and tags still pass; a blob or + // tree passes only if it is in its allowed set. Withheld and dangling blobs + // and trees (in the all-set, absent from allowed) drop. + let allowed_blobs: HashSet = ["b_pub".to_string()].into_iter().collect(); let all_blobs: HashSet = ["b_pub", "b_secret", "b_dangling"] .into_iter() .map(String::from) .collect(); + let allowed_trees: HashSet = + ["t_root", "t_pub"].into_iter().map(String::from).collect(); + let all_trees: HashSet = ["t_root", "t_pub", "t_secret", "t_dangling"] + .into_iter() + .map(String::from) + .collect(); let candidates = vec![ "commit1".to_string(), - "tree1".to_string(), + "tag1".to_string(), + "t_root".to_string(), + "t_pub".to_string(), + "t_secret".to_string(), + "t_dangling".to_string(), "b_pub".to_string(), "b_secret".to_string(), "b_dangling".to_string(), ]; - let got = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let got = replicable_objects_fail_closed( + candidates, + &allowed_blobs, + &all_blobs, + &allowed_trees, + &all_trees, + ); assert_eq!( got, vec![ "commit1".to_string(), - "tree1".to_string(), + "tag1".to_string(), + "t_root".to_string(), + "t_pub".to_string(), "b_pub".to_string() ] ); @@ -2482,8 +2604,12 @@ esac\n"; ); // Full-scan candidate set includes the dangling blob; fail-closed drops it. + // Empty tree sets keep today's keep-all-trees behavior for this blob-only + // case (U1 signature adapt; U3 wires the real tree walks). + let no_trees: HashSet = HashSet::new(); let candidates = vec![dangling_oid.clone(), public_oid.clone()]; - let replicable = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let replicable = + replicable_objects_fail_closed(candidates, &allowed, &all_blobs, &no_trees, &no_trees); assert!( !replicable.contains(&dangling_oid), "#99: a dangling private blob must not replicate" @@ -2494,6 +2620,210 @@ esac\n"; ); } + #[test] + fn withheld_object_set_contains_withheld_subtree_tree() { + // #172 repro as a test: the withheld /secret tree must enter the combined + // withheld set (today's blob-only walk omits it because trees are never in + // all_blob_oids). Nested secret/nested is the same leak class. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret/nested")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + std::fs::write(work.join("secret/nested/c.txt"), b"NESTED SECRET\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let nested_tree = oid("HEAD:secret/nested"); + let secret_blob = oid("HEAD:secret/b.txt"); + let public_tree = oid("HEAD:public"); + let public_blob = oid("HEAD:public/a.txt"); + let root_tree = oid("HEAD^{tree}"); + let head = oid("HEAD"); + let rules = [rule("/secret/**", &[])]; + let withheld = withheld_object_oids(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + withheld.contains(&secret_tree), + "withheld /secret tree must be in the combined withheld set (#172)" + ); + assert!( + withheld.contains(&nested_tree), + "nested withheld tree secret/nested must also be withheld" + ); + assert!( + withheld.contains(&secret_blob), + "withheld blob remains withheld" + ); + assert!(!withheld.contains(&root_tree), "root tree is allowed at /"); + assert!( + !withheld.contains(&public_tree), + "public subtree tree is not withheld" + ); + assert!( + !withheld.contains(&public_blob), + "public blob is not withheld" + ); + assert!(!withheld.contains(&head), "commits are never withheld"); + + let candidates_out = Command::new("git") + .args(["rev-list", "--objects", "--no-object-names", "HEAD"]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(candidates_out.status.success()); + let candidates: Vec = String::from_utf8_lossy(&candidates_out.stdout) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + let replicable = replicable_objects(candidates, &withheld); + assert!( + replicable.contains(&head) && replicable.contains(&root_tree), + "commit and root tree still replicate" + ); + assert!( + replicable.contains(&public_tree) && replicable.contains(&public_blob), + "public tree and blob still replicate" + ); + assert!( + !replicable.contains(&secret_tree) && !replicable.contains(&secret_blob), + "secret tree and blob must drop from the replicable set (#172)" + ); + assert!( + !replicable.contains(&nested_tree), + "nested withheld tree must drop too" + ); + } + + #[test] + fn withheld_object_set_blob_slice_matches_withheld_blob_oids() { + let (_td, bare, _s, _p) = fixture(); + let rules = [rule("/secret/**", &[])]; + let combined = withheld_object_oids(&bare, &rules, true, OWNER, None).unwrap(); + let blobs = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let all_blobs = + crate::git::push_delta::all_blob_oids(&bare, "git", Instant::now() + WALK_TIMEOUT) + .unwrap(); + let blob_slice: HashSet = combined.intersection(&all_blobs).cloned().collect(); + assert_eq!( + blob_slice, blobs, + "combined withheld set's blob slice must equal withheld_blob_oids (R4)" + ); + } + + #[test] + fn withheld_object_set_keeps_tree_shared_across_allowed_and_denied_paths() { + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + let rules = [rule("/sec/**", &[])]; + let withheld = withheld_object_oids(&work, &rules, true, OWNER, None).unwrap(); + assert!( + !withheld.contains(&pub_sub), + "a tree reachable at an allowed path is not withheld even when also at a denied path" + ); + } + + #[test] + fn withheld_object_set_empty_with_root_only_rules() { + let (_td, bare, _s, _p) = fixture(); + // A caller who has already passed the "/" gate: the listed reader of a + // root-only rule. Path-scoped withholding cannot apply, so the set is empty + // (R7, the has_path_scoped_rule short-circuit premise). + let rules = [rule("/", &["did:key:zFriend"])]; + let withheld = + withheld_object_oids(&bare, &rules, true, OWNER, Some("did:key:zFriend")).unwrap(); + assert!( + withheld.is_empty(), + "root-only rules cannot withhold a blob or tree from a caller who passed the / gate (R7)" + ); + } + + #[test] + fn withheld_object_walk_fails_closed_on_non_commit_ref() { + let (_td, bare, _s, _p) = fixture(); + let secret_tree = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:secret"]) + .current_dir(&bare) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(bare.join("refs/heads/treeref"), format!("{secret_tree}\n")).unwrap(); + let rules = [rule("/secret/**", &[])]; + let result = withheld_object_oids(&bare, &rules, true, OWNER, None); + assert!( + result.is_err(), + "a ref that peels to a tree must fail the withheld walk closed (Err), not a partial set" + ); + } + #[test] fn allowed_set_excludes_dangling_blob_for_every_caller() { // #126: a blob written via `git hash-object -w` but never referenced has diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3..3645b762 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -1602,7 +1602,7 @@ pub(crate) fn batch_budget_gate( /// `object_list` is the already-withheld-filtered OID set to pin: the caller /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling, so this -/// function never sees a withheld blob. `repo_path` is still needed to read each +/// function never sees a withheld blob or withheld-subtree tree. `repo_path` is still needed to read each /// object's bytes, and `git_bin` names the binary those reads run: the production /// callers pass the literal `"git"`, and a test passes a fake so the loop's own bound /// can be driven with a git that never answers. `repo_id` records the pin's provenance diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 14f1d582..4d125ec3 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -70,7 +70,8 @@ pub async fn pin_object( /// Pin any of the given candidate git objects that haven't yet been sent to /// Pinata. /// -/// `object_list` is the already-withheld-filtered OID set to pin: the caller +/// `object_list` is the already-withheld-filtered OID set to pin (blobs and +/// withheld-subtree trees already dropped): the caller /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes, and `git_bin` names the binary those