From 722a3b131195ad23455c49c0bebabbaf21fc1459 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:35:35 -0500 Subject: [PATCH] fix(node): drop withheld-subtree trees from the replication pin set A tree reachable only under a denied path survived both pin filters because trees were never in the blob universe, so public IPFS still learned child filenames and blob oids. Filter trees the same way as blobs on the delta withheld set and the full-scan allow-list. Closes #172. --- crates/gitlawb-node/src/api/repos.rs | 359 ++++++++++++++-- crates/gitlawb-node/src/git/push_delta.rs | 47 ++- .../gitlawb-node/src/git/visibility_pack.rs | 388 ++++++++++++++++-- crates/gitlawb-node/src/ipfs_pin.rs | 2 +- crates/gitlawb-node/src/pinata.rs | 3 +- 5 files changed, 716 insertions(+), 83 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c429..b617549fa 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -21,24 +21,24 @@ use crate::webhooks; /// The git all-zeros object id — the create/delete sentinel in a ref update. const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; -/// The set of blob OIDs withheld from **anonymous** replication for a repo, or -/// `None` when the repo must not replicate at all (private / mode A / -/// undetermined — fail closed). This is the anonymous replication gate: -/// `caller` is hard-coded to `None` and there is intentionally no caller -/// parameter, which distinguishes it from the per-caller read-serve projection -/// in `git_upload_pack` (which passes the real caller). Both the push pin path -/// and the reconciliation sweep call this helper so the two cannot drift on -/// what is withheld. `rules` is the already-fetched visibility-rule snapshot +/// The set of blob and withheld-subtree tree OIDs withheld from **anonymous** +/// replication for a repo, or `None` when the repo must not replicate at all +/// (private / mode A / undetermined, fail closed). This is the anonymous +/// replication gate: `caller` is hard-coded to `None` and there is intentionally +/// no caller parameter, which distinguishes it from the per-caller read-serve +/// projection in `git_upload_pack` (which passes the real caller). Both the push +/// pin path and the reconciliation sweep call this helper so the two cannot drift +/// on what is withheld. `rules` is the already-fetched visibility-rule snapshot /// (callers fetch once and may reuse it, e.g. for encrypt-then-pin). /// /// Returns `(announce, withheld)`: `announce` is whether the repo may be /// announced/replicated to the anonymous public at all (also gates gossip and -/// Arweave anchoring downstream), and `withheld` is the anonymous withheld blob -/// set when announceable (`None` when not announceable). A failed/panicked -/// withheld walk fails closed on both axes: `announce` is forced false and -/// `withheld` is `None`, so an unvetted push neither replicates blobs nor -/// announces. Returning both keeps the gate's announce decision a single -/// source rather than recomputing it at each call site. +/// Arweave anchoring downstream), and `withheld` is the anonymous withheld object +/// set (blobs and withheld-subtree trees) when announceable (`None` when not +/// announceable). A failed/panicked withheld walk fails closed on both axes: +/// `announce` is forced false and `withheld` is `None`, so an unvetted push +/// neither replicates objects nor announces. Returning both keeps the gate's +/// announce decision a single source rather than recomputing it at each call site. /// /// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): /// by the time the receive-pack tail calls this, the handler's write permit has @@ -64,13 +64,13 @@ async fn replication_withheld_set( } let withheld = match rules { // No path-scoped rule can withhold anything (covers the empty-rules and - // root-only-rules cases), so skip the full withheld_blob_oids walk and + // root-only-rules cases), so skip the full withheld_object_oids walk and // withhold nothing. The predicate's safety-invariant test guards that // this short-circuit matches what the walk would have returned. Some(rules) if !visibility_pack::has_path_scoped_rule(&rules) => { Some(std::collections::HashSet::new()) } - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep + // withheld_object_oids walks every ref with blocking `git ls-tree`; keep // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); @@ -83,18 +83,18 @@ async fn replication_withheld_set( // The permit lives inside the blocking closure: a started walk // always completes holding it. let _permit = permit; - crate::git::visibility_pack::withheld_blob_oids_bounded( + crate::git::visibility_pack::withheld_object_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await .map_err(|e| { - tracing::warn!(err = %e, "withheld_blob_oids task panicked; skipping replication") + tracing::warn!(err = %e, "withheld_object_oids task panicked; skipping replication") }) .ok() .and_then(|r| { r.map_err(|e| { - tracing::warn!(err = %e, "withheld_blob_oids failed; skipping replication") + tracing::warn!(err = %e, "withheld_object_oids failed; skipping replication") }) .ok() }) @@ -111,16 +111,17 @@ async fn replication_withheld_set( } } -/// The replicable object set for a full-scan pin fallback, failing closed (#99). +/// The replicable object set for a full-scan pin fallback, failing closed (#99, #172). /// The full-scan candidate set includes dangling objects the reachable-only /// withheld set never classified, so compute the reachable visibility-allowed -/// blob set and the all-blob universe off the async worker and keep only -/// non-blobs plus allowed blobs. Any error in either walk (or a task panic) +/// blob set, the reachable visibility-allowed tree set, and the all-blob plus +/// all-tree universes off the async worker and keep only commits/tags plus +/// allowed blobs plus allowed trees. Any error in any walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. /// /// Always walks (there is no no-git arm), so the whole blocking scan runs under -/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// one `git_encrypt_semaphore` admission permit (#174 F4): see /// `acquire_scan_permit` for the defer rationale and the honest residuals. #[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( @@ -139,15 +140,16 @@ async fn fail_closed_full_scan_objects( crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; - // One whole-scan deadline shared across both phases (#174 F4). A fresh - // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus - // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the - // deadline caps total occupancy at ~1x: phase 1 runs against the remaining - // budget, and if it consumes the budget phase 2 gets what is left and fails - // closed (pins nothing) rather than over-holding — the safe direction. The cost - // is honest: a genuinely large repo whose phase 1 nears the budget under-pins - // this push rather than the previous silent ~2x hold; size the budget so both - // phases normally fit. + // One whole-scan deadline shared across every phase (#174 F4). A fresh + // `Instant::now() + timeout` for a later phase let a large-but-successful + // earlier phase plus a full later phase hold the scan permit ~Nx the + // configured budget. Sharing the deadline caps total occupancy at ~1x: + // each phase runs against the remaining budget, and if an earlier phase + // consumes the budget a later one gets what is left and fails closed + // (pins nothing) rather than over-holding, the safe direction. The cost + // is honest: a genuinely large repo whose first phase nears the budget + // under-pins this push rather than the previous silent ~2x hold; size the + // budget so the blob walk, tree walk, and typed enumeration normally fit. let deadline = std::time::Instant::now() + timeout; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, @@ -157,19 +159,32 @@ async fn fail_closed_full_scan_objects( is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; + let allowed_trees = crate::git::visibility_pack::replicable_tree_set_bounded( + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, + )?; + let (all_blobs, all_trees) = + crate::git::push_delta::all_blob_and_tree_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( - candidates, &allowed, &all_blobs, + candidates, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, )) }) .await .map_err(|e| { - tracing::warn!(err = %e, "fail-closed blob walk task panicked; pinning nothing this push") + tracing::warn!(err = %e, "fail-closed object walk task panicked; pinning nothing this push") }) .ok() .and_then(|r| { r.map_err(|e| { - tracing::warn!(err = %e, "fail-closed blob walk failed; pinning nothing this push") + tracing::warn!(err = %e, "fail-closed object walk failed; pinning nothing this push") }) .ok() }) @@ -2444,10 +2459,10 @@ async fn post_receive_replication_tail( did: String, ) { // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means this push pins nothing (private / mode A / - // undetermined, or a walk that failed): skip every pin so even commit and tree - // objects (which withheld_blob_oids never lists) stay local. Fail closed: a + // may read this repo at all and, if so, which blob and withheld-subtree tree + // OIDs must not leave the node. `withheld == None` means this push pins nothing + // (private / mode A / undetermined, or a walk that failed): skip every pin so + // even commit and tag objects (which the withheld walk never lists) stay local. Fail closed: a // private or undetermined repo never leaks. The announce decision that gates // the network-facing sends is taken separately, below. let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); @@ -3883,6 +3898,229 @@ mod tests { ); } + /// #172: first-push delta (empty old tips, `full_scan == false`) must drop the + /// withheld-subtree tree. The withheld set comes from `replication_withheld_set`, + /// the production wrapper all three delta call sites already use. + #[tokio::test] + async fn first_push_delta_drops_withheld_subtree_tree() { + use std::process::Command; + use std::time::Duration; + + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &std::path::Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let head = oid("HEAD"); + let root_tree = oid("HEAD^{tree}"); + let public_tree = oid("HEAD:public"); + let public_blob = oid("HEAD:public/a.txt"); + let secret_tree = oid("HEAD:secret"); + let secret_blob = oid("HEAD:secret/b.txt"); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(64)); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + bare.clone(), + vec![head.clone()], + vec![], + "git".into(), + Duration::from_secs(600), + false, + ) + .await; + assert!( + !pin_set.full_scan, + "a first push of a commit tip must take the delta path, not full-scan" + ); + assert!( + pin_set.candidates.contains(&secret_tree) && pin_set.candidates.contains(&secret_blob), + "precondition: the first-push delta includes the withheld tree and blob, \ + so a later drop is a filter hit, not a missing-candidate skip" + ); + + let (_announce, withheld) = replication_withheld_set( + sem, + Some(vec![vis_rule("/secret/**", &[])]), + OWNER_DID, + true, + bare, + "git".into(), + Duration::from_secs(600), + ) + .await; + let withheld = withheld + .expect("announceable public repo with a path-scoped rule must yield a withheld set"); + let replicable = + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld); + assert!( + replicable.contains(&head) && replicable.contains(&root_tree), + "commit and root tree still replicate" + ); + assert!( + replicable.contains(&public_tree) && replicable.contains(&public_blob), + "public tree and blob still replicate" + ); + assert!( + !replicable.contains(&secret_tree), + "withheld-subtree tree must not survive the first-push delta pin set (#172)" + ); + assert!( + !replicable.contains(&secret_blob), + "withheld blob still drops" + ); + } + + /// #172: full-scan fallback drops a withheld-subtree tree and a dangling tree, + /// keeping the commit, root tree, public tree, and public blob. + #[tokio::test] + async fn full_scan_drops_withheld_and_dangling_trees() { + use std::io::Write; + use std::process::Command; + use std::time::Duration; + + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &std::path::Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let head = oid("HEAD"); + let root_tree = oid("HEAD^{tree}"); + let public_tree = oid("HEAD:public"); + let public_blob = oid("HEAD:public/a.txt"); + let secret_tree = oid("HEAD:secret"); + let secret_blob = oid("HEAD:secret/b.txt"); + + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_blob}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling_tree = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let deadline = std::time::Instant::now() + Duration::from_secs(600); + let candidates = crate::git::push_delta::list_all_objects(&bare, "git", deadline).unwrap(); + assert!( + candidates.contains(&dangling_tree), + "precondition: dangling tree is in the full-scan universe" + ); + assert!( + candidates.contains(&secret_tree) && candidates.contains(&secret_blob), + "precondition: full-scan universe includes the withheld tree and blob" + ); + + let replicable = fail_closed_full_scan_objects( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + bare, + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + "git".into(), + Duration::from_secs(600), + ) + .await; + assert!(replicable.contains(&head), "HEAD commit kept"); + assert!(replicable.contains(&root_tree), "root tree kept"); + assert!(replicable.contains(&public_tree), "public tree kept"); + assert!(replicable.contains(&public_blob), "public blob kept"); + assert!( + !replicable.contains(&secret_tree), + "withheld-subtree tree dropped on full-scan (#172)" + ); + assert!( + !replicable.contains(&secret_blob), + "withheld blob dropped on full-scan" + ); + assert!( + !replicable.contains(&dangling_tree), + "dangling tree dropped on full-scan (#172, #99 geometry for trees)" + ); + } + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + @@ -6859,7 +7097,7 @@ mod tests { // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget // would let it finish. - let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) case \"$*\" in *-rzt*) : ;; *) sleep 1.5 ;; esac ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; let git_bin = write_fake_git(tmp.path(), body); // A candidate that is NOT a blob (never appears in all_blob_oids): kept by // replicable_objects_fail_closed only if phase 2 actually ran to completion. @@ -6885,6 +7123,45 @@ mod tests { ); } + /// #172 / #174 F4 sibling: a successful blob walk that nearly exhausts the + /// shared budget must leave the tree allowed-set walk only the remainder, so + /// the scan pins nothing. Distinguishes `ls-tree -rzt` (tree) from `ls-tree -rz` + /// (blob). Load-bearing together with a fresh timeout on both the tree walk and + /// the typed enumeration: those two later phases completing independently would + /// keep the non-blob non-tree candidate. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_with_tree_allowed_set_walk() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Blob ls-tree -rz sleeps 1.5s; tree ls-tree -rzt sleeps 1.5s; cat-file is + // instant. With a 2s whole-scan budget the tree walk is reaped. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) case \"$*\" in *-rzt*) sleep 1.5 ;; *) sleep 1.5 ;; esac ;;\n cat-file) : ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful blob walk must leave the tree allowed-set walk only \ + the SHARED remainder, so it reaps and the scan fails closed; got {objs:?}" + ); + } + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full /// object-id list while walking it, so concurrent pin loops across many repos must /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 0b5696933..e13b65af8 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -19,7 +19,7 @@ //! objects the reachable withheld set never classified, so subtracting that set //! is not enough — a dangling private blob would slip through (#99). The caller //! signals a full scan via [`PinCandidateSet::full_scan`] and must then apply -//! the fail-closed blob filter (`visibility_pack::replicable_objects_fail_closed`) +//! the fail-closed blob-and-tree filter (`visibility_pack::replicable_objects_fail_closed`) //! instead of the plain reachable-only subtraction. //! //! Because the pin candidate set needs only the OID *set* (never the per-path @@ -211,8 +211,9 @@ pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> R /// Like [`list_all_objects`] but pairs each OID with its object type, via /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed -/// filter needs to tell blobs (content, withholdable) from commits/trees -/// (structural, never withheld) without typing the candidate list itself. +/// filter needs to tell blobs (content, withholdable) and trees (content, +/// withholdable since #135/#173) from commits/tags (structural, never withheld) +/// without typing the candidate list itself. pub fn list_all_objects_with_type( repo_path: &Path, git_bin: &str, @@ -244,24 +245,48 @@ pub fn list_all_objects_with_type( /// Every blob OID in the repo, including unreachable/dangling ones. The /// fail-closed pin filter drops any candidate blob absent from the reachable, /// visibility-allowed set; a dangling private blob is in this set but not the -/// allowed set, so it never replicates (#99). +/// allowed set, so it never replicates (#99). Production uses +/// [`all_blob_and_tree_oids`] (one typed pass); this wrapper stays for tests. +#[cfg(test)] pub fn all_blob_oids( repo_path: &Path, git_bin: &str, deadline: Instant, ) -> Result> { - Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? - .into_iter() - .filter(|(_, ty)| ty == "blob") - .map(|(oid, _)| oid) - .collect()) + Ok(all_blob_and_tree_oids(repo_path, git_bin, deadline)?.0) +} + +/// Every blob OID and every tree OID in the repo, including unreachable/dangling +/// ones, from one typed enumeration. The fail-closed pin filter drops any +/// candidate blob or tree absent from the matching reachable visibility-allowed +/// set; a dangling or withheld object is in this universe but not the allowed +/// set, so it never replicates (#99, #172). +pub fn all_blob_and_tree_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result<(HashSet, HashSet)> { + let mut blobs = HashSet::new(); + let mut trees = HashSet::new(); + for (oid, ty) in list_all_objects_with_type(repo_path, git_bin, deadline)? { + match ty.as_str() { + "blob" => { + blobs.insert(oid); + } + "tree" => { + trees.insert(oid); + } + _ => {} + } + } + Ok((blobs, trees)) } /// The pin candidate OIDs for a push plus whether they came from a full-repo /// scan. `full_scan` is true when the delta could not be used and the whole /// object DB (including dangling objects) was enumerated — the caller must then -/// apply the fail-closed blob filter, because the reachable-only withheld set -/// cannot classify dangling blobs (#99). +/// apply the fail-closed blob-and-tree filter, because the reachable-only withheld set +/// cannot classify dangling blobs or trees (#99, #172). #[derive(Debug, Clone, PartialEq, Eq)] pub struct PinCandidateSet { pub candidates: Vec, diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 086669947..3a5373a8c 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -1,7 +1,9 @@ -//! Resolve which blob OIDs must be withheld from a caller because every path -//! at which the blob appears is denied by the repo's visibility rules. Trees -//! and commits are never withheld (mode B keeps SHAs intact); only blob -//! content is held back. +//! Resolve which object OIDs must be withheld from a caller because every path +//! at which the object appears is denied by the repo's visibility rules. Blobs +//! and subtree trees are withholdable (mode B; trees since #135/#173). Commits +//! and tags are never withheld: they are root-level metadata a "/" reader already +//! clears. The replication pin path uses [`withheld_object_oids_bounded`] so a +//! withheld-subtree tree is not exported to public IPFS (#172). use crate::db::VisibilityRule; use crate::visibility::{visibility_check, Decision}; @@ -540,6 +542,79 @@ pub fn withheld_blob_oids_bounded( )) } +/// Reachable blob AND withheld-subtree tree OIDs the anonymous (or `caller`) +/// replication audience must not receive. The single source for +/// `replication_withheld_set`: a blob-only set handed to [`replicable_objects`] +/// reintroduces #172 (withheld trees pin to public IPFS). +/// +/// Strict where [`tree_paths`] is lenient. This feeds a withheld filter, so +/// absence means replicate; a missed reachable tree would leak. The walk +/// therefore matches [`blob_paths`]: [`assert_all_refs_are_commits`] first, then +/// a failing `rev-list --all [HEAD]` is an error, then [`object_paths`] (already +/// fail-closed on ls-tree / non-UTF-8) plus [`root_tree_pairs`]. [`tree_paths`] +/// stays the lenient allow-list walk for GET /ipfs/{cid} and the full-scan +/// allowed-tree set, where absence withholds. +pub fn withheld_object_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits: Vec = String::from_utf8_lossy(&commits_out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + let triples = object_paths(repo_path, git_bin, &commits, deadline)?; + let mut pairs: Vec<(String, String)> = triples + .into_iter() + .filter(|(_, _, kind)| kind == "blob" || kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + pairs.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(withheld_from_pairs( + &pairs, rules, is_public, owner_did, caller, + )) +} + +/// [`withheld_object_oids_bounded`] with the test walk budget. +#[cfg(test)] +pub fn withheld_object_oids( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + withheld_object_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + /// Withheld set from an already-computed (oid, "/path") listing: a blob is /// withheld only when visibility denies the caller at *every* path it appears /// at. Split out so a caller that already walked `blob_paths` (e.g. @@ -588,8 +663,9 @@ pub fn has_path_scoped_rule(rules: &[VisibilityRule]) -> bool { /// Objects that may replicate to the public: everything not in `withheld`. /// Order-preserving. The single seam every replication site (IPFS, Pinata) -/// passes its object list through; option B would later reroute the withheld -/// ones through encrypt-then-pin instead of dropping them. +/// passes its object list through on the delta path. `withheld` MUST be the +/// combined blob-and-tree set from [`withheld_object_oids_bounded`] (via +/// `replication_withheld_set`); a blob-only set reintroduces #172. pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec { all.into_iter() .filter(|oid| !withheld.contains(oid)) @@ -627,6 +703,23 @@ pub fn replicable_blob_set_bounded( ) } +/// Reachable tree OIDs that visibility ALLOWS the anonymous replication +/// audience at some path. The tree analog of [`replicable_blob_set_bounded`], +/// used on the fail-closed full-scan pin path so a withheld or dangling tree +/// is absent and dropped (#172). +pub fn replicable_tree_set_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -827,11 +920,12 @@ fn root_tree_pairs( /// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` /// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every /// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the -/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree -/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed -/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two -/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], -/// bounded by the same shared `deadline`. +/// reachable-commit set ONCE (leniently, see [`reachable_commit_oids`]; the tree +/// allowed-set feeds the `/ipfs/{cid}` tree gate AND the full-scan pin allow-list, +/// where absence = fail-closed) and drives both the ls-tree walk and the root-tree +/// pass from it, so the two cannot diverge and neither re-enumerates. The tree analog +/// of [`blob_paths`], bounded by the same shared `deadline`. Not for a withheld +/// filter: that path is [`withheld_object_oids_bounded`]. fn tree_paths( repo_path: &Path, git_bin: &str, @@ -1150,22 +1244,29 @@ pub fn reachable_commit_tag_oids_bounded( Ok(set) } -/// Objects safe to replicate, failing closed on blobs (#99). A candidate -/// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are -/// structural, never content-withheld) OR it is in `allowed_blobs` (reachable -/// and visibility-allowed). This drops both withheld reachable blobs and -/// dangling/unreachable blobs the reachable walk never classified, without -/// tagging the candidate list with per-object types. Used on the full-scan pin -/// path, where the candidate set can contain dangling objects the reachable-only -/// withheld set cannot cover; the delta path keeps `replicable_objects`. +/// Objects safe to replicate, failing closed on blobs AND trees (#99, #172). +/// A candidate replicates iff it is neither a known blob nor a known tree +/// (commits and tags: structural, a "/" reader already clears) OR it is in +/// `allowed_blobs` OR it is in `allowed_trees`. Trees have been content-withholdable +/// since #135/#173; the old premise that they always pass is falsified. A dangling +/// or withheld tree is in `all_tree_oids` and absent from `allowed_trees`, so it +/// drops. Used on the full-scan pin path, where the candidate set can contain +/// dangling objects the reachable-only withheld set cannot cover; the delta path +/// keeps [`replicable_objects`]. pub fn replicable_objects_fail_closed( candidates: Vec, allowed_blobs: &HashSet, all_blob_oids: &HashSet, + allowed_trees: &HashSet, + all_tree_oids: &HashSet, ) -> Vec { candidates .into_iter() - .filter(|oid| !all_blob_oids.contains(oid) || allowed_blobs.contains(oid)) + .filter(|oid| { + let is_blob = all_blob_oids.contains(oid); + let is_tree = all_tree_oids.contains(oid); + (!is_blob && !is_tree) || allowed_blobs.contains(oid) || allowed_trees.contains(oid) + }) .collect() } @@ -2384,28 +2485,49 @@ esac\n"; } #[test] - fn fail_closed_keeps_nonblobs_and_allowed_blobs_only() { - // Non-blob objects (commit/tree) always pass; a blob passes only if it - // is in the allowed set. A withheld blob and a dangling blob (both in - // all_blob_oids, neither in allowed) are dropped. - let allowed: HashSet = ["b_pub".to_string()].into_iter().collect(); + fn fail_closed_keeps_commits_allowed_blobs_and_allowed_trees_only() { + // Deliberate invariant flip of fail_closed_keeps_nonblobs_and_allowed_blobs_only + // (#172). That test's premise ("commits and trees are structural, never + // content-withheld") was falsified by #135/#173: a withheld-subtree tree is + // content, same as a withheld blob. Commits and tags still pass; a blob or + // tree passes only if it is in its allowed set. Withheld and dangling blobs + // and trees (in the all-set, absent from allowed) drop. + let allowed_blobs: HashSet = ["b_pub".to_string()].into_iter().collect(); let all_blobs: HashSet = ["b_pub", "b_secret", "b_dangling"] .into_iter() .map(String::from) .collect(); + let allowed_trees: HashSet = + ["t_root", "t_pub"].into_iter().map(String::from).collect(); + let all_trees: HashSet = ["t_root", "t_pub", "t_secret", "t_dangling"] + .into_iter() + .map(String::from) + .collect(); let candidates = vec![ "commit1".to_string(), - "tree1".to_string(), + "tag1".to_string(), + "t_root".to_string(), + "t_pub".to_string(), + "t_secret".to_string(), + "t_dangling".to_string(), "b_pub".to_string(), "b_secret".to_string(), "b_dangling".to_string(), ]; - let got = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let got = replicable_objects_fail_closed( + candidates, + &allowed_blobs, + &all_blobs, + &allowed_trees, + &all_trees, + ); assert_eq!( got, vec![ "commit1".to_string(), - "tree1".to_string(), + "tag1".to_string(), + "t_root".to_string(), + "t_pub".to_string(), "b_pub".to_string() ] ); @@ -2482,8 +2604,12 @@ esac\n"; ); // Full-scan candidate set includes the dangling blob; fail-closed drops it. + // Empty tree sets keep today's keep-all-trees behavior for this blob-only + // case (U1 signature adapt; U3 wires the real tree walks). + let no_trees: HashSet = HashSet::new(); let candidates = vec![dangling_oid.clone(), public_oid.clone()]; - let replicable = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let replicable = + replicable_objects_fail_closed(candidates, &allowed, &all_blobs, &no_trees, &no_trees); assert!( !replicable.contains(&dangling_oid), "#99: a dangling private blob must not replicate" @@ -2494,6 +2620,210 @@ esac\n"; ); } + #[test] + fn withheld_object_set_contains_withheld_subtree_tree() { + // #172 repro as a test: the withheld /secret tree must enter the combined + // withheld set (today's blob-only walk omits it because trees are never in + // all_blob_oids). Nested secret/nested is the same leak class. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret/nested")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + std::fs::write(work.join("secret/nested/c.txt"), b"NESTED SECRET\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let nested_tree = oid("HEAD:secret/nested"); + let secret_blob = oid("HEAD:secret/b.txt"); + let public_tree = oid("HEAD:public"); + let public_blob = oid("HEAD:public/a.txt"); + let root_tree = oid("HEAD^{tree}"); + let head = oid("HEAD"); + let rules = [rule("/secret/**", &[])]; + let withheld = withheld_object_oids(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + withheld.contains(&secret_tree), + "withheld /secret tree must be in the combined withheld set (#172)" + ); + assert!( + withheld.contains(&nested_tree), + "nested withheld tree secret/nested must also be withheld" + ); + assert!( + withheld.contains(&secret_blob), + "withheld blob remains withheld" + ); + assert!(!withheld.contains(&root_tree), "root tree is allowed at /"); + assert!( + !withheld.contains(&public_tree), + "public subtree tree is not withheld" + ); + assert!( + !withheld.contains(&public_blob), + "public blob is not withheld" + ); + assert!(!withheld.contains(&head), "commits are never withheld"); + + let candidates_out = Command::new("git") + .args(["rev-list", "--objects", "--no-object-names", "HEAD"]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(candidates_out.status.success()); + let candidates: Vec = String::from_utf8_lossy(&candidates_out.stdout) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + let replicable = replicable_objects(candidates, &withheld); + assert!( + replicable.contains(&head) && replicable.contains(&root_tree), + "commit and root tree still replicate" + ); + assert!( + replicable.contains(&public_tree) && replicable.contains(&public_blob), + "public tree and blob still replicate" + ); + assert!( + !replicable.contains(&secret_tree) && !replicable.contains(&secret_blob), + "secret tree and blob must drop from the replicable set (#172)" + ); + assert!( + !replicable.contains(&nested_tree), + "nested withheld tree must drop too" + ); + } + + #[test] + fn withheld_object_set_blob_slice_matches_withheld_blob_oids() { + let (_td, bare, _s, _p) = fixture(); + let rules = [rule("/secret/**", &[])]; + let combined = withheld_object_oids(&bare, &rules, true, OWNER, None).unwrap(); + let blobs = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let all_blobs = + crate::git::push_delta::all_blob_oids(&bare, "git", Instant::now() + WALK_TIMEOUT) + .unwrap(); + let blob_slice: HashSet = combined.intersection(&all_blobs).cloned().collect(); + assert_eq!( + blob_slice, blobs, + "combined withheld set's blob slice must equal withheld_blob_oids (R4)" + ); + } + + #[test] + fn withheld_object_set_keeps_tree_shared_across_allowed_and_denied_paths() { + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + let rules = [rule("/sec/**", &[])]; + let withheld = withheld_object_oids(&work, &rules, true, OWNER, None).unwrap(); + assert!( + !withheld.contains(&pub_sub), + "a tree reachable at an allowed path is not withheld even when also at a denied path" + ); + } + + #[test] + fn withheld_object_set_empty_with_root_only_rules() { + let (_td, bare, _s, _p) = fixture(); + // A caller who has already passed the "/" gate: the listed reader of a + // root-only rule. Path-scoped withholding cannot apply, so the set is empty + // (R7, the has_path_scoped_rule short-circuit premise). + let rules = [rule("/", &["did:key:zFriend"])]; + let withheld = + withheld_object_oids(&bare, &rules, true, OWNER, Some("did:key:zFriend")).unwrap(); + assert!( + withheld.is_empty(), + "root-only rules cannot withhold a blob or tree from a caller who passed the / gate (R7)" + ); + } + + #[test] + fn withheld_object_walk_fails_closed_on_non_commit_ref() { + let (_td, bare, _s, _p) = fixture(); + let secret_tree = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:secret"]) + .current_dir(&bare) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(bare.join("refs/heads/treeref"), format!("{secret_tree}\n")).unwrap(); + let rules = [rule("/secret/**", &[])]; + let result = withheld_object_oids(&bare, &rules, true, OWNER, None); + assert!( + result.is_err(), + "a ref that peels to a tree must fail the withheld walk closed (Err), not a partial set" + ); + } + #[test] fn allowed_set_excludes_dangling_blob_for_every_caller() { // #126: a blob written via `git hash-object -w` but never referenced has diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3b..3645b7620 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -1602,7 +1602,7 @@ pub(crate) fn batch_budget_gate( /// `object_list` is the already-withheld-filtered OID set to pin: the caller /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling, so this -/// function never sees a withheld blob. `repo_path` is still needed to read each +/// function never sees a withheld blob or withheld-subtree tree. `repo_path` is still needed to read each /// object's bytes, and `git_bin` names the binary those reads run: the production /// callers pass the literal `"git"`, and a test passes a fake so the loop's own bound /// can be driven with a git that never answers. `repo_id` records the pin's provenance diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 14f1d5824..4d125ec34 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -70,7 +70,8 @@ pub async fn pin_object( /// Pin any of the given candidate git objects that haven't yet been sent to /// Pinata. /// -/// `object_list` is the already-withheld-filtered OID set to pin: the caller +/// `object_list` is the already-withheld-filtered OID set to pin (blobs and +/// withheld-subtree trees already dropped): the caller /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes, and `git_bin` names the binary those