From 162dd3391ec7946c80e967ac08c94932e6684380 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Thu, 27 Aug 2026 13:43:44 +0600 Subject: [PATCH 1/4] feat(node): implement reconciliation sweep as durability backstop (#218) Reconciliation sweep: periodic background worker that re-derives pin/seal sets and fills gaps so a dropped replication job never means data loss. - visibility_pack.rs: all_object_paths (two-phase ls-tree + cat-file --batch-all-objects), allowed_blob_tree_sets_bounded, ObjectPath/BlobTreeSets type aliases - db/mod.rs: atomic policy transactions, get/set_node_state, has_ipfs_cid, filter_ipfs_pinned_oids, filter_pinata_pinned_oids, repo_policy_epoch, migration v27 (node_state table + policy_epoch column) - ipfs_pin.rs: PolicyFence struct, dispatch fence re-check before POST - pinata.rs: dispatch fence re-check before POST - encrypted_pin.rs: dispatch fence re-check - reconciliation.rs: full sweep worker with keyset cursor, PolicyFence - api/ipfs.rs: effective_cid fallback in list_pins - main.rs: spawn reconciliation sweep worker - .cargo/audit.toml: RUSTSEC-2026-0258 h2 ignore (pending #368) --- .cargo/audit.toml | 12 + .env.example | 14 + Cargo.lock | 1 + README.md | 1 + crates/gitlawb-attest/src/attestation.rs | 31 + crates/gitlawb-core/src/identity.rs | 40 + crates/gitlawb-node/Cargo.toml | 1 + crates/gitlawb-node/src/api/ipfs.rs | 33 +- crates/gitlawb-node/src/api/repos.rs | 31 +- crates/gitlawb-node/src/config.rs | 11 + crates/gitlawb-node/src/db/mod.rs | 587 +++++- crates/gitlawb-node/src/encrypted_pin.rs | 385 +++- crates/gitlawb-node/src/git/push_delta.rs | 43 +- crates/gitlawb-node/src/git/store.rs | 5 + .../gitlawb-node/src/git/visibility_pack.rs | 244 ++- crates/gitlawb-node/src/ipfs_pin.rs | 223 ++- crates/gitlawb-node/src/main.rs | 24 + crates/gitlawb-node/src/metrics.rs | 57 +- crates/gitlawb-node/src/pinata.rs | 35 + crates/gitlawb-node/src/reconciliation.rs | 1703 +++++++++++++++++ docs/RUN-A-NODE.md | 1 + 21 files changed, 3399 insertions(+), 83 deletions(-) create mode 100644 crates/gitlawb-node/src/reconciliation.rs diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 309fac34f..3c917fdda 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -35,4 +35,16 @@ ignore = [ # ever built with the `mysql` feature, or any other consumer of rsa enters # the build, at which point it becomes a real reachable advisory. "RUSTSEC-2023-0071", # rsa Marvin attack (no fix; not linked in our build) + + # h2 0.4.13 (unbounded empty DATA frames DoS). Present in Cargo.lock because + # reqwest/hyper transitively depend on h2. The fix requires h2 >=0.4.16. + # REMOVE once #368 lands with a compatible h2 update. + "RUSTSEC-2026-0258", # h2 unbounded empty DATA frames + + # lru 0.16.4 (use-after-free in pop()). Reachable via alloy -> alloy-provider. + # No fix available: alloy 1.7.3 pins alloy-provider which uses lru 0.16.4. + # The lru 0.12.5 advisory (RUSTSEC-2026-0253) is also present (via aws-sdk-s3) + # but that was fixed by reverting aws-sdk-s3 upgrade (we kept the older version + # to avoid the h2 issue). REMOVE once alloy updates its lru dependency. + "RUSTSEC-2026-0253", # lru use-after-free (both 0.12.5 and 0.16.4) ] diff --git a/.env.example b/.env.example index 81c60824d..552968517 100644 --- a/.env.example +++ b/.env.example @@ -305,6 +305,20 @@ GITLAWB_TRUSTED_PROXY= # Enable automatic background sync from known peers GITLAWB_AUTO_SYNC=false +# ── Reconciliation sweep ───────────────────────────────────────────────── +# Periodic durability sweep: re-derives the public pin set and the withheld-blob +# recovery set each hour and fills gaps so a dropped replication job never means +# data loss. Defaults to true; set to false to disable the sweep even when a pin +# backend (IPFS/Pinata) is configured. +# +# Phase-capability matrix: +# - Public pin repair: IPFS-only, Pinata-only, or both (requires the +# respective backend to be configured). +# - Encrypted recovery repair: requires local IPFS (GITLAWB_IPFS_API). +# Pinata-only nodes reconcile public pins only; encrypted recovery +# reconciliation is not performed. +GITLAWB_RECONCILIATION_SWEEP=true + # ── iCaptcha proof-of-intelligence gate ─────────────────────────────────── # Optional gate on create_repo + register: require callers to present an # iCaptcha proof (X-ICaptcha-Proof header) earned at icaptcha.gitlawb.com. diff --git a/Cargo.lock b/Cargo.lock index 3f29b0767..d202730d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3468,6 +3468,7 @@ dependencies = [ "mockito", "multiaddr", "prometheus", + "rand 0.8.6", "reqwest", "serde", "serde_json", diff --git a/README.md b/README.md index 3a092bf21..c4ad07875 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. Defaults to `false` during the staged rollout below. | | `GITLAWB_ENFORCE_OWNER_PUSH` | Require the authenticated pusher to be the repo owner on `git-receive-pack`. **Defaults to `true`.** A `did:key` signature is authentication, not authorization — anyone can mint a key and sign — so with this off every signed caller may push to every repository, private ones included. Delegated and CI keys count as non-owners: a UCAN `git/push` capability is verified but not yet honored for authorization, so they cannot push while this is on. Set `false` only for a rolling upgrade; see [`docs/RUN-A-NODE.md`](docs/RUN-A-NODE.md). | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | +| `GITLAWB_RECONCILIATION_SWEEP` | Enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Public pin repair runs against any configured backend (IPFS, Pinata, or both). Encrypted recovery repair requires local IPFS (`GITLAWB_IPFS_API`); Pinata-only nodes reconcile public pins only. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | diff --git a/crates/gitlawb-attest/src/attestation.rs b/crates/gitlawb-attest/src/attestation.rs index 0e29e3a90..88ab2fdae 100644 --- a/crates/gitlawb-attest/src/attestation.rs +++ b/crates/gitlawb-attest/src/attestation.rs @@ -483,6 +483,37 @@ mod tests { assert!(matches!(err, Error::Signature(_))); } + /// The identity-point forgery must be rejected: the shared attestation + /// verifier is a cert-bound provenance gate, so accepting the weak-key + /// signature would let anyone mint a forged attestation that verifies. + /// Strict verification rejects small-order public keys and R (the identity + /// point here), which ordinary verification does not. + #[test] + fn verify_rejects_identity_point_forgery() { + let cert_hash = sample_cert_hash(); + let mut att = dummy_attestation(&fresh(), cert_hash); + + // Public key A = identity point (0,1); signature R = identity, S = 0. + // The equation `[S]B = R + [k]A` then holds for any k and any message. + let identity = [ + 1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]; + let mut buf = Vec::with_capacity(34); + buf.extend_from_slice(&ED25519_MULTICODEC); + buf.extend_from_slice(&identity); + att.signer = format!( + "did:key:{}", + multibase::encode(multibase::Base::Base58Btc, &buf) + ); + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&identity); + att.sig = B64U.encode(sig); + + let err = att.verify_signature(cert_hash).unwrap_err(); + assert!(matches!(err, Error::Signature(_))); + } + /// A payload that happens to contain a `cert_hash` field of its own does /// not interfere with the outer binding: the attestation envelope's /// `cert_hash` is the only field consulted by `verify_signature`, and the diff --git a/crates/gitlawb-core/src/identity.rs b/crates/gitlawb-core/src/identity.rs index beef4d1bc..ca87f8ecc 100644 --- a/crates/gitlawb-core/src/identity.rs +++ b/crates/gitlawb-core/src/identity.rs @@ -77,6 +77,14 @@ impl Keypair { } /// Verify an Ed25519 signature. +/// +/// Strict verification: rejects small-order `R` and small-order public keys +/// (the identity point, and any point of low order). Ordinary `verify` accepts +/// a signature forged with the identity point as the public key plus +/// `R = identity, S = 0`, which verifies for *any* message. `identity::verify` +/// is the shared primitive behind HTTP request authentication, UCANs, and +/// certificates, so weak-key acceptance is an authentication bypass, not a +/// malleability nuance. pub fn verify(verifying_key: &VerifyingKey, msg: &[u8], sig_bytes: &[u8; 64]) -> Result<()> { let sig = Signature::from_bytes(sig_bytes); verifying_key @@ -208,6 +216,38 @@ mod tests { ); } + /// The identity-point forgery: with public key A = identity, R = identity, + /// and S = 0, the equation `[S]B = R + [k]A` holds for every message, + /// because `[k]·identity = identity`. Ordinary (non-strict) Ed25519 + /// verification accepts it, so the shared `verify` primitive must use + /// strict verification, which rejects small-order R and public keys. + #[test] + fn verify_rejects_identity_point_forgery() { + use ed25519_dalek::Verifier; + // The identity point (0,1) compresses to y = 1 with sign bit 0. + let identity = [ + 1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]; + let vk = VerifyingKey::from_bytes(&identity).expect("identity point is on the curve"); + let mut sig_bytes = [0u8; 64]; + sig_bytes[..32].copy_from_slice(&identity); + let msg = b"arbitrary message the key owner never signed"; + + // Prove the forged signature satisfies the ordinary verification + // equation, so the strict check below is what actually defends the + // boundary (not a signature that was already invalid everywhere). + assert!( + vk.verify(msg, &Signature::from_bytes(&sig_bytes)).is_ok(), + "identity-point forgery must satisfy ordinary verification (this is why strict is needed)" + ); + + assert!( + verify(&vk, msg, &sig_bytes).is_err(), + "strict verification must reject the identity-point forgery" + ); + } + #[test] fn verify_rejects_weak_key_signature() { // Regression guard for strict verification: a signature forged under a diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index c583569cb..65b18ba49 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -73,6 +73,7 @@ alloy = { version = "1", default-features = false, features = [ "rpc-types-eth", ] } libp2p-dns = { version = "0.44.0", features = ["tokio"] } +rand = { workspace = true } [dev-dependencies] mockito = "1" diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 92d129803..4ba33122f 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -2130,14 +2130,41 @@ async fn gate_and_serve( /// GET /api/v1/ipfs/pins /// -/// Returns all CIDs that have been pinned to the local IPFS node from git -/// objects received via push. Each entry includes the git SHA-256 hex, the -/// CIDv1 string, and the timestamp when it was pinned. +/// Returns all CIDs that have been pinned from git objects received via push. +/// Each entry includes the git SHA-256 hex, a CIDv1 string, and the timestamp +/// when it was pinned. For Pinata-only rows (no local IPFS pin), the `cid` +/// field carries `pinata_cid` so CLI consumers see a usable value. +/// +/// Rows with neither a local nor a Pinata CID are omitted so the response +/// only contains rows with at least one backend. Both `cid` (local IPFS) and +/// `pinata_cid` (Pinata) are nullable: a row with only `cid` set is local-only, +/// a row with only `pinata_cid` set is remote-only, and a row with both has +/// been replicated to both backends. pub async fn list_pins(State(state): State) -> Result> { // Bare `?` so connection-class sqlx failures downcast to `AppError::Db` and // map to 503 `db_unavailable` (not 500 via `.map_err(AppError::Internal)`) (#251). let pins = state.db.list_pinned_cids().await?; + let pins: Vec = pins + .into_iter() + .filter(|p| p.cid.is_some() || p.pinata_cid.is_some()) + .map(|p| { + // Backward compatibility: `cid` in the response is the local CID + // when present, falling back to the Pinata CID for remote-only rows. + // Clients like `gl ipfs list` read only `pin["cid"]`; a NULL here + // would render as "?". Both provenance fields are always included so + // consumers can distinguish local-only, remote-only, and dual rows. + let effective_cid = p.cid.as_deref().or(p.pinata_cid.as_deref()); + serde_json::json!({ + "sha256_hex": p.sha256_hex, + "cid": effective_cid, + "local_cid": p.cid, + "pinata_cid": p.pinata_cid, + "pinned_at": p.pinned_at, + }) + }) + .collect(); + Ok(Json(serde_json::json!({ "pins": pins, "count": pins.len(), diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c429..7abaacbbd 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -149,17 +149,17 @@ async fn fail_closed_full_scan_objects( // this push rather than the previous silent ~2x hold; size the budget so both // phases normally fit. let deadline = std::time::Instant::now() + timeout; - let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( - &disk_path, - &git_bin, - deadline.saturating_duration_since(std::time::Instant::now()), - &rules, - is_public, - &owner_did, - )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; + let (allowed, allowed_trees, all_blobs, all_trees) = + crate::git::visibility_pack::allowed_blob_tree_sets_bounded( + &disk_path, + &git_bin, + deadline, + &rules, + is_public, + &owner_did, + )?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( - candidates, &allowed, &all_blobs, + candidates, &allowed, &all_blobs, &allowed_trees, &all_trees, )) }) .await @@ -1403,6 +1403,7 @@ async fn pin_new_objects_gated( db, repo_id, batch_budget, + None, ) .await } @@ -1471,7 +1472,14 @@ async fn pin_and_encrypt_objects( &ctx.db, repo_id, &node_seed, + // The real git, not `ctx.git_bin`: tests point that at a fake + // walk git, and the seal reads must run the real one. + "git", + crate::ipfs_pin::PIN_BATCH_BUDGET, &recipients, + // Push path: recipients derived at admission under a write lease, + // no sweep-style snapshot to fence (see PolicyFence's doc). + None, ) .await; @@ -2731,6 +2739,9 @@ async fn post_receive_replication_tail( &db_clone, &repo_id, crate::ipfs_pin::PIN_BATCH_BUDGET, + // Push path: no sweep-style batch snapshot to fence (see + // PolicyFence's doc). + None, ) .await, ) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 1fbf4376e..2a001d14e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -129,6 +129,17 @@ pub struct Config { #[arg(long, env = "GITLAWB_AUTO_SYNC", default_value_t = false)] pub auto_sync: bool, + /// Enable the periodic reconciliation sweep that re-derives pin/seal sets + /// and fills durability gaps. Defaults to true; set to false to disable + /// the sweep even when a pin backend (IPFS/Pinata) is configured. + #[arg( + long, + env = "GITLAWB_RECONCILIATION_SWEEP", + default_value_t = true, + action = clap::ArgAction::Set + )] + pub reconciliation_sweep: bool, + /// Irys URL for Arweave permanent anchoring. /// Leave empty to disable. Use https://devnet.irys.xyz for free devnet. #[arg(long, env = "GITLAWB_IRYS_URL", default_value = "")] diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index cc2cf0bd7..a88f9b5e5 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1,8 +1,9 @@ +use std::time::Duration; + use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use sqlx::{postgres::PgPoolOptions, PgPool, Row}; -use std::time::Duration; use tracing::info; use uuid::Uuid; @@ -172,7 +173,9 @@ pub struct RepoReplica { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PinnedCidRecord { pub sha256_hex: String, - pub cid: String, + /// Local IPFS CID. NULL for Pinata-only rows where the object was never + /// fetched by this node's IPFS instance. + pub cid: Option, pub pinned_at: String, pub pinata_cid: Option, } @@ -1540,6 +1543,36 @@ impl Db { Ok(rows.into_iter().map(row_to_repo).collect()) } + /// Like `list_all_repos_deduped` but ordered by a stable key (`id`) so a + /// keyset cursor deterministically covers every repo regardless of push + /// activity. Used by the reconciliation sweep to avoid starving idle repos. + /// Only `limit` rows are returned; pass `cursor = None` for the first page. + pub async fn list_all_repos_deduped_stable( + &self, + cursor: Option<&str>, + limit: i64, + ) -> Result> { + let sql = format!( + "{} + SELECT d.id, d.name, d.owner_did, d.description, d.is_public, + d.default_branch, d.created_at, d.updated_at, d.disk_path, + d.forked_from, d.machine_id + FROM deduped d + WHERE ($2::text IS NULL OR d.id > $2::text) + ORDER BY d.id ASC + LIMIT $3", + Self::dedup_cte() + ); + let rows = sqlx::query(&sql) + .bind(None::<&str>) + .bind(cursor) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + Ok(rows.into_iter().map(row_to_repo).collect()) + } + /// Repos currently quarantined (admitted as mirrors but withheld from every /// listing surface). `list_all_repos_deduped` excludes these (its `DEDUP_CTE` /// filters `quarantined = FALSE`), so a gate that resolves a slug against the @@ -1712,18 +1745,26 @@ impl Db { .unwrap_or(false)) } - /// Set or clear a repo's quarantine flag. Returns the number of rows touched - /// (0 if no such repo). Backs the (deferred) operator release surface; the - /// admission path writes the flag via `upsert_mirror_repo`. Allowed dead - /// outside tests until the operator endpoint lands. + /// Set or clear a repo's quarantine flag and bump the policy epoch + /// atomically. Returns the number of rows touched (0 if no such repo). + /// A failure in either statement rolls back both. #[cfg_attr(not(test), allow(dead_code))] pub async fn set_repo_quarantine(&self, repo_id: &str, quarantined: bool) -> Result { + let mut tx = self.pool.begin().await?; let result = sqlx::query("UPDATE repos SET quarantined = $1 WHERE id = $2") .bind(quarantined) .bind(repo_id) - .execute(&self.pool) + .execute(&mut *tx) .await?; - Ok(result.rows_affected()) + let affected = result.rows_affected(); + if affected > 0 { + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(affected) } /// Repo ids currently quarantined, for operator review. Allowed dead outside @@ -2763,6 +2804,47 @@ impl Db { } } +// ── Node state ──────────────────────────────────────────────────────────────── + +impl Db { + /// Read an opaque node-state value. Returns `None` when the key has never + /// been written. Used by the reconciliation sweep to persist its keyset + /// cursor across restarts (R2-P1). + pub async fn get_node_state(&self, key: &str) -> Result> { + let row = sqlx::query("SELECT value FROM node_state WHERE key = $1") + .bind(key) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get("value"))) + } + + /// Write an opaque node-state value (upsert). `None` deletes the key so a + /// cleared cursor does not accumulate stale rows. + pub async fn set_node_state(&self, key: &str, value: Option<&str>) -> Result<()> { + match value { + Some(v) => { + sqlx::query( + "INSERT INTO node_state (key, value, updated_at) + VALUES ($1, $2, $3) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = EXCLUDED.updated_at", + ) + .bind(key) + .bind(v) + .bind(Utc::now().to_rfc3339()) + .execute(&self.pool) + .await?; + } + None => { + sqlx::query("DELETE FROM node_state WHERE key = $1") + .bind(key) + .execute(&self.pool) + .await?; + } + } + Ok(()) + } +} + // ── Pinned CIDs ─────────────────────────────────────────────────────────────── impl Db { @@ -3387,16 +3469,38 @@ impl Db { ) .fetch_all(&self.pool) .await?; - Ok(rows - .into_iter() - .filter(|r| gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid"))) - .map(|r| PinnedCidRecord { + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + if !gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid")) { + continue; + } + out.push(PinnedCidRecord { sha256_hex: r.get("sha256_hex"), - cid: r.get("cid"), + // `try_get::>` maps only SQL NULL to None (a + // Pinata-only row); a corrupt `cid` column surfaces as a decode + // error through `?` instead of being silently misread as a + // Pinata-only row. The old `try_get().ok()` conflated the two. + cid: r.try_get("cid")?, pinned_at: r.get("pinned_at"), pinata_cid: r.get("pinata_cid"), - }) - .collect()) + }); + } + Ok(out) + } + + /// Returns true when this object has a real local IPFS CID. After migration + /// v27 cleared legacy `cid = pinata_cid` fallback rows (provenance is now + /// recorded, never inferred), `cid IS NOT NULL` is the complete predicate. + pub async fn has_ipfs_cid(&self, sha256_hex: &str) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) as cnt FROM pinned_cids + WHERE sha256_hex = $1 + AND cid IS NOT NULL", + ) + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(row.get::("cnt") > 0) } /// Returns true if this object already has a Pinata CID recorded. @@ -3410,10 +3514,60 @@ impl Db { Ok(row.get::("cnt") > 0) } + /// Given a list of sha256_hex values, returns the subset that already have + /// a Pinata CID recorded. Used by the reconciliation sweep to skip objects + /// that Pinata has already handled. Chunked like `filter_ipfs_pinned_oids` + /// to bound the `ANY($1)` array size on full uncapped object lists (R1-P3). + pub async fn filter_pinata_pinned_oids(&self, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(Vec::new()); + } + const CHUNK_SIZE: usize = 1000; + let mut out = Vec::new(); + for chunk in oids.chunks(CHUNK_SIZE) { + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = ANY($1) AND pinata_cid IS NOT NULL", + ) + .bind(chunk) + .fetch_all(&self.pool) + .await?; + out.extend(rows.into_iter().map(|r| r.get("sha256_hex"))); + } + Ok(out) + } + + /// Given a list of sha256_hex values, returns the subset that have a real + /// local IPFS CID (`cid IS NOT NULL`; after migration v27 provenance is + /// recorded, never inferred from CID inequality). Used by the reconciliation + /// sweep to skip IPFS-complete objects. + /// + /// The input is processed in fixed-size chunks so the `ANY($1)` array sent + /// to Postgres is bounded even when the sweep hands over a full uncapped + /// object list (R1-P3). + pub async fn filter_ipfs_pinned_oids(&self, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(Vec::new()); + } + const CHUNK_SIZE: usize = 1000; + let mut out = Vec::new(); + for chunk in oids.chunks(CHUNK_SIZE) { + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids + WHERE sha256_hex = ANY($1) + AND cid IS NOT NULL", + ) + .bind(chunk) + .fetch_all(&self.pool) + .await?; + out.extend(rows.into_iter().map(|r| r.get("sha256_hex"))); + } + Ok(out) + } + /// Record the Pinata CID for a git object. /// /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, - /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// CIDv1/raw/sha-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb @@ -4044,6 +4198,9 @@ impl Db { // ── Path-scoped Visibility ──────────────────────────────────────────────────── impl Db { + /// Set or replace a visibility rule and bump the repo's policy epoch + /// atomically. A sweep reading the rule after it commits must see the new + /// epoch; the two are never visible from different transactions. pub async fn set_visibility_rule( &self, repo_id: &str, @@ -4055,6 +4212,7 @@ impl Db { let id = Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); let readers = serde_json::to_string(reader_dids).unwrap_or_else(|_| "[]".to_string()); + let mut tx = self.pool.begin().await?; sqlx::query( "INSERT INTO visibility_rules (id, repo_id, path_glob, mode, reader_dids, created_by, created_at) @@ -4072,20 +4230,43 @@ impl Db { .bind(&readers) .bind(created_by) .bind(&now) - .execute(&self.pool) + .execute(&mut *tx) .await?; + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; Ok(()) } + /// Remove a visibility rule and bump the repo's policy epoch atomically. pub async fn remove_visibility_rule(&self, repo_id: &str, path_glob: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; sqlx::query("DELETE FROM visibility_rules WHERE repo_id = $1 AND path_glob = $2") .bind(repo_id) .bind(path_glob) - .execute(&self.pool) + .execute(&mut *tx) + .await?; + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } + /// Current visibility-policy epoch for a repo (0 for a repo with no entry). + /// The epoch is bumped by every rule or quarantine mutation, so a value that + /// changes between two reads proves a policy change happened in between. + pub async fn repo_policy_epoch(&self, repo_id: &str) -> Result { + let row = sqlx::query("SELECT policy_epoch FROM repos WHERE id = $1") + .bind(repo_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("policy_epoch")).unwrap_or(0)) + } + pub async fn list_visibility_rules(&self, repo_id: &str) -> Result> { let rows = sqlx::query( "SELECT id, repo_id, path_glob, mode, reader_dids, created_by, created_at @@ -5094,6 +5275,376 @@ mod migration_tests { assert_eq!(attempted_at_of(&db, "z6Mkfoo/failed").await, None); assert_eq!(attempted_at_of(&db, "z6Mkfoo/done").await, None); } + + /// Migration v12 makes pinned_cids.cid nullable so record_pinata_cid can + /// create Pinata-only rows without a local IPFS CID. This test seeds a + /// pre-v12 schema (cid NOT NULL, pinata_cid column exists but no + /// nullability change yet) with rows in each of the three states the + /// has_ipfs_cid / filter_ipfs_pinned_oids predicates must classify: + /// + /// (1) cid IS NOT NULL, pinata_cid IS NULL → has_ipfs = true + /// (2) cid IS NOT NULL, cid != pinata_cid → has_ipfs = true + /// (3) cid IS NOT NULL, cid = pinata_cid (legacy) → has_ipfs = false + /// + /// Legacy row (3) stops being a special case because migration v27 clears + /// `cid = pinata_cid` back to NULL, so `has_ipfs_cid` reduces to the plain + /// `cid IS NOT NULL` predicate (provenance recorded, never inferred). + /// + /// After the migration we also test that a Pinata-only INSERT (cid = NULL) + /// works and produces has_ipfs = false, has_pinata = true. + #[sqlx::test] + async fn migration_v12_makes_cid_nullable_and_preserves_classification(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + + // Create all tables, then drop the NOT NULL constraint on cid + // and drop schema_migrations records to simulate a pre-v12 node. + db.migrate().await.unwrap(); + sqlx::query("ALTER TABLE pinned_cids ALTER COLUMN cid SET NOT NULL") + .execute(&db.pool) + .await + .unwrap(); + + sqlx::query("DELETE FROM schema_migrations") + .execute(&db.pool) + .await + .unwrap(); + for m in MIGRATIONS.iter().take_while(|m| m.version < 12) { + sqlx::query( + "INSERT INTO schema_migrations (version, name, applied_at) + VALUES ($1, $2, $3)", + ) + .bind(m.version) + .bind(m.name) + .bind("2026-07-01T00:00:00Z") + .execute(&db.pool) + .await + .unwrap(); + } + + // ── Seed legacy rows ─────────────────────────────────────────── + let now = "2026-07-01T12:00:00Z"; + + // (1) Real local IPFS pin, no Pinata. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_real_only") + .bind("QmRealLocalCid") + .bind(now) + .bind(Option::<&str>::None) + .execute(&db.pool) + .await + .unwrap(); + + // (2) Both CIDs present and distinct. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_both_distinct") + .bind("QmLocalForThisBlob") + .bind(now) + .bind("QmPinataForThisBlob") + .execute(&db.pool) + .await + .unwrap(); + + // (3) Legacy row where cid was set to pinata_cid as fallback. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_legacy_fallback") + .bind("QmLegacyEqual") + .bind(now) + .bind("QmLegacyEqual") + .execute(&db.pool) + .await + .unwrap(); + + // ── Apply migration v12 ──────────────────────────────────────── + db.migrate().await.unwrap(); + + // ── Assertions ───────────────────────────────────────────────── + + // Column is now nullable. + let nullable: String = sqlx::query_scalar( + "SELECT is_nullable FROM information_schema.columns + WHERE table_name = 'pinned_cids' AND column_name = 'cid'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(nullable, "YES", "cid must be nullable after v12"); + + // Classification: has_ipfs_cid. + assert!( + db.has_ipfs_cid("sha_real_only").await.unwrap(), + "real local IPFS CID must be classified as pinned" + ); + assert!( + db.has_ipfs_cid("sha_both_distinct").await.unwrap(), + "distinct local CID must be classified as pinned" + ); + assert!( + !db.has_ipfs_cid("sha_legacy_fallback").await.unwrap(), + "legacy equal-cid row must NOT be classified as having an IPFS CID" + ); + + // has_pinata_cid. + assert!( + !db.has_pinata_cid("sha_real_only").await.unwrap(), + "no pinata_cid means has_pinata = false" + ); + assert!( + db.has_pinata_cid("sha_both_distinct").await.unwrap(), + "non-null pinata_cid means has_pinata = true" + ); + assert!( + db.has_pinata_cid("sha_legacy_fallback").await.unwrap(), + "non-null pinata_cid means has_pinata = true (legacy row)" + ); + + // ── Pinata-only INSERT (new post-v12 row) ────────────────────── + db.record_pinata_cid("sha_pinata_only", "QmPinataOnly") + .await + .unwrap(); + assert!( + !db.has_ipfs_cid("sha_pinata_only").await.unwrap(), + "Pinata-only row must NOT be classified as having a local IPFS CID" + ); + assert!( + db.has_pinata_cid("sha_pinata_only").await.unwrap(), + "Pinata-only row must have has_pinata = true" + ); + + // ── Idempotent re-run ────────────────────────────────────────── + db.migrate().await.unwrap(); + } + + /// Migration v27 clears legacy rows where cid was set to pinata_cid as a + /// fallback, so `has_ipfs_cid` no longer has to infer provenance from CID + /// inequality (R2-P2). Rows where the CIDs genuinely differ are untouched. + #[sqlx::test] + async fn migration_v27_clears_legacy_equal_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + // Seed one legacy equal-cid row and one distinct-cid row, then mark + // v27 (and v28, applied after it) as not yet run so re-running + // migrate() exercises the backfill in isolation. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_equal', 'QmSame', $1, 'QmSame'), + ('sha_distinct', 'QmLocal', $1, 'QmPinata')", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version >= 27") + .execute(&db.pool) + .await + .unwrap(); + + db.migrate().await.unwrap(); + + // Backfilled row now has no local CID; distinct row is untouched. + assert!( + !db.has_ipfs_cid("sha_equal").await.unwrap(), + "legacy equal-cid row must be cleared to NULL by v27" + ); + assert!( + db.has_ipfs_cid("sha_distinct").await.unwrap(), + "distinct-cid row must survive the backfill" + ); + assert!(db.has_pinata_cid("sha_equal").await.unwrap()); + } + + /// `list_pinned_cids` must map a SQL NULL `cid` (Pinata-only row) to + /// `None`. The old `try_get("cid").ok()` conflated NULL with a decode + /// failure, so `/api/v1/ipfs/pins` could silently omit or misrepresent a + /// row instead of surfacing the DB error. + #[sqlx::test] + async fn list_pinned_cids_maps_null_cid_to_none(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + // One row with a real CID, one Pinata-only row (cid NULL). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_real', 'QmReal', $1, 'QmPinata'), + ('sha_pinata_only', NULL, $1, 'QmPinata2')", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + + let pins = db.list_pinned_cids().await.unwrap(); + let real = pins + .iter() + .find(|p| p.sha256_hex == "sha_real") + .expect("real-cid row must be listed"); + assert_eq!(real.cid.as_deref(), Some("QmReal")); + let pinata_only = pins + .iter() + .find(|p| p.sha256_hex == "sha_pinata_only") + .expect("Pinata-only row must be listed"); + assert_eq!(pinata_only.cid, None, "NULL cid must map to None"); + } + + /// A corrupt `cid` value must surface as a decode error, not a silent + /// None. Postgres only stores values of the column's declared type, so + /// reach the decode failure by retyping the column to bytea (a future + /// migration doing the same is the realistic corruption path). The column + /// is retyped before the first `list_pinned_cids` call so the query plan + /// is compiled against the corrupt type. + #[sqlx::test] + async fn list_pinned_cids_errors_on_corrupt_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + sqlx::query("ALTER TABLE pinned_cids ALTER COLUMN cid TYPE bytea USING NULL::bytea") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_bad', E'\\\\xdeadbeef', $1, NULL)", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + + let err = db + .list_pinned_cids() + .await + .expect_err("corrupt cid column must fail the whole listing"); + assert!( + err.to_string().contains("invalid type") || err.to_string().contains("cid"), + "decode failure must be the reported error, got: {err}" + ); + } + + /// Migration v28 creates the node_state key/value table and the get/set + /// helpers round-trip through it (used by the sweep cursor persistence). + #[sqlx::test] + async fn node_state_roundtrip_and_delete(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + None, + "absent key reads as None" + ); + + db.set_node_state("sweep_cursor", Some("repo/b")) + .await + .unwrap(); + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + Some("repo/b".to_string()), + "value survives a write + read" + ); + + // Upsert overwrites. + db.set_node_state("sweep_cursor", Some("repo/c")) + .await + .unwrap(); + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + Some("repo/c".to_string()) + ); + + // None deletes the key. + db.set_node_state("sweep_cursor", None).await.unwrap(); + assert_eq!(db.get_node_state("sweep_cursor").await.unwrap(), None); + } + + /// record_pinned_cid must repair a stale WRONG local CID, not only fill a + /// NULL or Pinata-fallback slot (R1-P2): an object pinned once with the + /// wrong bytes is overwritten by a subsequent push-path pin, but the sweep + /// gap filter (`cid IS NOT NULL`) excludes rows with a present CID from + /// re-processing, so the sweep cannot repair them. + #[sqlx::test] + async fn record_pinned_cid_repairs_stale_wrong_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + // A stale wrong CID that is neither NULL nor equal to pinata_cid. + db.record_pinned_cid("sha_stale", "QmStaleWrong") + .await + .unwrap(); + db.record_pinata_cid("sha_stale", "QmPinataX") + .await + .unwrap(); + + // Re-pin with the correct CID — must overwrite despite the existing + // distinct cid column. + db.record_pinned_cid("sha_stale", "QmCorrect") + .await + .unwrap(); + + let cid: String = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_stale'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, "QmCorrect", "stale wrong CID must be repaired"); + } + + /// record_pinata_cid must clear a legacy cid = pinata_cid fallback (v27's + /// belt-and-suspenders) so a later Pinata-only row is never misread as a + /// local IPFS pin. + #[sqlx::test] + async fn record_pinata_cid_clears_legacy_equal_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + db.record_pinned_cid("sha_fallback", "QmFallback") + .await + .unwrap(); + // Simulate a legacy row where cid was forced equal to pinata_cid. + sqlx::query( + "UPDATE pinned_cids SET pinata_cid = 'QmFallback' WHERE sha256_hex = 'sha_fallback'", + ) + .execute(&db.pool) + .await + .unwrap(); + + // Recording a new (different) Pinata CID must NULL the stale fallback cid. + db.record_pinata_cid("sha_fallback", "QmPinataNew") + .await + .unwrap(); + + let cid: Option = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_fallback'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, None, "legacy equal-cid fallback must be cleared"); + + // But a genuine local pin plus a distinct Pinata CID is preserved. + db.record_pinned_cid("sha_genuine", "QmLocalGenuine") + .await + .unwrap(); + db.record_pinata_cid("sha_genuine", "QmPinataGenuine") + .await + .unwrap(); + let cid: String = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_genuine'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, "QmLocalGenuine"); + } } #[cfg(test)] diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 19b80651b..a9fc42714 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -6,6 +6,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::Path; use std::str::FromStr; +use std::time::Duration; use ed25519_dalek::VerifyingKey; use gitlawb_core::did::Did; @@ -106,17 +107,60 @@ fn plan_seal(node_seed: &[u8; 32], dids: &BTreeSet, stored_tag: Option<& /// `node_seed` keys the opaque recipients tag. Returns `(oid, cid)` for each blob /// actually sealed and recorded this call (the per-push delta), used by Option B3 /// to anchor a manifest. Recipient identities are never stored or returned. +/// +/// Nine args (the fence joins the seal's eight) but grouping them would churn +/// both callers and the race/hung-git tests for no behavioral gain. +#[allow(clippy::too_many_arguments)] pub async fn encrypt_and_pin( ipfs_api: &str, repo_path: &Path, db: &Db, repo_id: &str, node_seed: &[u8; 32], + git_bin: &str, + batch_budget: Duration, recipients: &HashMap>, + fence: Option<&crate::ipfs_pin::PolicyFence>, ) -> Vec<(String, String)> { let mut sealed = Vec::new(); let mut skipped_unresolvable = 0usize; - for (oid, dids) in recipients { + // One shared read deadline for the whole batch, like `pin_new_objects`: a + // hung git child is watchdog-reaped at this bound, so the outer + // `PIN_PHASE_DEADLINE` timeout cannot be held open by a blocking read + // (R1-P2). Each read runs under `spawn_blocking` — it is synchronous child + // spawn + pipe drain + watchdog join. + let read_deadline = std::time::Instant::now() + batch_budget; + let total = recipients.len(); + for (attempted, (oid, dids)) in recipients.iter().enumerate() { + // Batch budget gate (R2-P3), mirroring the public pin loops: an object + // is never started with a remainder too small to cover a bounded read's + // teardown. This is consistency (the seal is bounded by the outer + // `PIN_PHASE_DEADLINE` either way), but it keeps the three loops from + // drifting apart in how they report a truncated batch. + if crate::ipfs_pin::batch_budget_gate( + "encrypted-seal", + read_deadline, + sealed.len(), + total - attempted, + ) + .is_none() + { + break; + } + // Policy fence (R1-P1): the recipients snapshot was derived before the + // long withheld-blob walk; if a visibility rule moved while that walk + // ran (a reader added or removed), stop sealing instead of pinning to a + // stale recipient set. Checked FIRST so a changed policy costs nothing. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + oid = %oid, + "visibility policy changed after the recipients snapshot; stopping the seal loop" + ); + break; + } + } // A DB read failure is not a cache miss: re-sealing here would do an // avoidable IPFS write during a partial outage. Skip and retry next push. let stored_tag = match db.encrypted_blob_recipients_tag(repo_id, oid).await { @@ -152,7 +196,9 @@ pub async fn encrypt_and_pin( } SealPlan::Seal { keys, tag } => (keys, tag), }; - let data = match crate::git::store::read_object(repo_path, oid) { + let data = match read_object_bounded_spawn_blocking(git_bin, repo_path, oid, read_deadline) + .await + { Ok(Some((_t, bytes))) => bytes, Ok(None) => { tracing::warn!(oid = %oid, "git object not found; skipping encrypted pin"); @@ -170,6 +216,22 @@ pub async fn encrypt_and_pin( continue; } }; + // Dispatch fence (R1-P1): re-read the policy epoch immediately before + // the irreversible HTTP POST. The iteration-top check catches a narrow + // that landed before work began; THIS check catches a narrow that landed + // during the tag lookup, recipient resolution, git read, or seal — all + // of which can take seconds. Without this, a reader removed during + // preparation can still receive a newly published envelope. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + oid = %oid, + "visibility policy changed during encrypted seal preparation; aborting upload" + ); + break; + } + } let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await { Ok(c) if !c.is_empty() => c, Ok(_) => { @@ -201,10 +263,33 @@ pub async fn encrypt_and_pin( sealed } +/// Bounded, reaped git object read for the seal loop, run off the async thread: +/// `read_object_bounded` is synchronous child spawn + pipe drain + watchdog +/// join, so blocking the runtime task on it would let a hung git hold a worker +/// thread (R1-P2). The `deadline` is the batch's shared read deadline; a child +/// still alive at it is SIGTERM/SIGKILL group-reaped by the watchdog. +async fn read_object_bounded_spawn_blocking( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> anyhow::Result)>> { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let sha256_hex = sha256_hex.to_string(); + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha256_hex, deadline) + .map_err(anyhow::Error::from) + }) + .await + .map_err(|e| anyhow::anyhow!("read_object spawn_blocking join failed: {e}"))? +} + #[cfg(test)] mod tests { use super::*; use ed25519_dalek::SigningKey; + use std::time::Duration; fn did_key(seed: u8) -> String { let vk = SigningKey::from_bytes(&[seed; 32]).verifying_key(); @@ -359,4 +444,300 @@ mod tests { other => panic!("changed recipient set must re-seal; got {other:?}"), } } + + /// A reader removed mid-seal must stop the seal loop (R1-P1 "race test for + /// reader removal"): `encrypt_and_pin` re-checks the policy fence before + /// each blob, so a `remove_visibility_rule` landing while the first seal is + /// in flight aborts before a later blob is pinned to a stale recipient set. + #[sqlx::test] + async fn encrypt_and_pin_stops_sealing_when_reader_removed_mid_batch(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("seal-race.git"); + + // Three loose blobs, each withheld (path-scoped deny exists so the sweep + // would have derived recipients for them). + let oids: Vec = { + crate::git::store::init_bare(&repo_path).expect("init bare repo"); + (0..3) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("secret blob {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + }; + + // A real repos row so the fence has an epoch and a reader can be removed. + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "seal-race-repo".into(), + owner_did: "did:key:zSealRaceOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + // A rule whose removal is the "reader removed" mutation: one reader per + // blob, all under the same path glob. + let reader = did_key(1); + db.set_visibility_rule( + &repo_id, + "**/secret/*", + crate::db::VisibilityMode::B, + std::slice::from_ref(&reader), + "did:key:zSealRaceOwner", + ) + .await + .expect("set rule"); + + // IPFS endpoint that delays the FIRST add 2s so the removal lands while + // that seal is in flight, then answers immediately. + let endpoint = delaying_cid_endpoint(vec![Duration::from_secs(2)]).await; + + let recipients: HashMap> = oids + .iter() + .cloned() + .map(|oid| { + let mut s = BTreeSet::new(); + s.insert(reader.clone()); + (oid, s) + }) + .collect(); + + let fence = crate::ipfs_pin::PolicyFence::capture(&db, &repo_id) + .await + .expect("fence captures"); + + let sealed = tokio::time::timeout(Duration::from_secs(30), async { + let seal_db = db.clone(); + let seal_repo = repo_path.clone(); + let seal_endpoint = endpoint.clone(); + let seal_repo_id = repo_id.clone(); + let handle = tokio::spawn(async move { + encrypt_and_pin( + &seal_endpoint, + &seal_repo, + &seal_db, + &seal_repo_id, + &SEED, + "git", + Duration::from_secs(60), + &recipients, + Some(&fence), + ) + .await + }); + // Let the first add start (endpoint sleeps 2s), then remove the + // reader so the fence is stale before the loop checks again. + tokio::time::sleep(Duration::from_millis(300)).await; + db.remove_visibility_rule(&repo_id, "**/secret/*") + .await + .expect("remove rule"); + handle.await.expect("seal task") + }) + .await + .expect("wedge guard: the fence abort must not take 30s"); + + assert!( + sealed.len() < oids.len(), + "a reader removal landing mid-batch must abort before every blob is sealed: {}", + sealed.len() + ); + assert!( + !sealed.is_empty(), + "at least the blob already in flight before the removal completes" + ); + } + + /// A hung git must not hold the seal loop past its read budget (R1-P2): the + /// git read runs under `spawn_blocking` against `read_object_bounded`, so + /// the watchdog reaps a wedged child at the batch deadline and the loop + /// keeps its shape instead of blocking a runtime worker indefinitely. + #[cfg(unix)] + #[sqlx::test] + async fn encrypt_and_pin_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("seal-hung.git"); + let oids: Vec = { + crate::git::store::init_bare(&repo_path).expect("init bare repo"); + (0..2) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("secret blob {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + }; + + // A git that wedges forever, ignoring SIGTERM, so only the watchdog's + // SIGKILL can reap it. + let fake = tmp.path().join("hanging-git"); + std::fs::write(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n").unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "seal-hung-repo".into(), + owner_did: "did:key:zSealHungOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + db.set_visibility_rule( + &repo_id, + "**/secret/*", + crate::db::VisibilityMode::B, + &[did_key(1)], + "did:key:zSealHungOwner", + ) + .await + .expect("set rule"); + + let recipients: HashMap> = oids + .iter() + .cloned() + .map(|oid| { + let mut s = BTreeSet::new(); + s.insert(did_key(1)); + (oid, s) + }) + .collect(); + + // Unreachable endpoint: even if a read somehow succeeded, the pin would + // fail; the read itself is the thing under test. + let started = std::time::Instant::now(); + let sealed = tokio::time::timeout( + Duration::from_secs(60), + encrypt_and_pin( + "http://127.0.0.1:9", + &repo_path, + &db, + &repo_id, + &SEED, + fake.to_str().unwrap(), + Duration::from_secs(2), + &recipients, + None, + ), + ) + .await + .expect("a hung git must not hold the seal past the outer wedge guard"); + + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(10), + "a hung git must be watchdog-reaped inside the read budget, not block the loop for ~10s+ (took {elapsed:?})" + ); + assert!( + sealed.is_empty(), + "with a hung git no blob can be read, so nothing may be reported sealed" + ); + } + + /// Local TCP endpoint that answers `{ "Hash": "QmMock" }` after an optional + /// per-request delay, so a seal can be made to straddle a policy mutation. + async fn delaying_cid_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let body = br#"{"Hash":"QmSealRaceMockCid"}"#; + let _ = sock + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + .as_bytes(), + ) + .await; + let _ = sock.write_all(body).await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } } diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 0b5696933..e4b971d80 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -209,10 +209,42 @@ pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> R .collect()) } +/// The set of objects reachable from any ref, via +/// `git rev-list --all --objects --no-object-names`. +/// +/// The full-object-database enumeration ([`list_all_objects`]) contains +/// dangling commits, trees, and blobs (`git cat-file --batch-all-objects` lists +/// loose objects from an aborted or still-running push). Blob candidates are +/// already fail-closed against the reachable, visibility-allowed set — but +/// commits and trees have no path scoping to fail closed against, so the sweep +/// must bound them to ref-reachability or an unreferenced commit's message, +/// author, and parent links (and any unreferenced tree) would be published to a +/// public IPFS/Pinata endpoint. This is that reachability bound. +pub fn reachable_object_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["rev-list", "--all", "--objects", "--no-object-names"], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); + Ok(stdout + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + /// Like [`list_all_objects`] but pairs each OID with its object type, via /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. +#[allow(dead_code)] // used by tests and all_blob_oids pub fn list_all_objects_with_type( repo_path: &Path, git_bin: &str, @@ -245,6 +277,7 @@ pub fn list_all_objects_with_type( /// fail-closed pin filter drops any candidate blob absent from the reachable, /// visibility-allowed set; a dangling private blob is in this set but not the /// allowed set, so it never replicates (#99). +#[allow(dead_code)] // used by visibility_pack tests pub fn all_blob_oids( repo_path: &Path, git_bin: &str, @@ -278,10 +311,12 @@ pub struct PinCandidateSet { /// Every degraded path is **logged**, not silent: a full-scan fallback, a /// failed full scan, and a panicked blocking task each emit a warning. On a /// failed full scan or a task panic the candidate set is empty (pin nothing -/// this push); that is a durability gap the reconciliation sweep backstops, and -/// it can never leak because the withheld/fail-closed filter still runs on -/// whatever set is returned. `full_scan` rides on the returned set so the caller -/// knows when the dangling-inclusive filter is required. +/// this push); that is a durability gap the reconciliation sweep backstops +/// when it is enabled and a pin backend is configured (a node running with the +/// sweep disabled or with no IPFS/Pinata backend has no backstop), and it can +/// never leak because the withheld/fail-closed filter still runs on whatever +/// set is returned. `full_scan` rides on the returned set so the caller knows +/// when the dangling-inclusive filter is required. /// /// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, /// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b4198..bd8e19859 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -273,6 +273,9 @@ pub struct TreeEntry { /// /// Get just the object type. Returns `None` if the object doesn't exist; a /// probe that could not examine the object store is `Err`, never `None`. +// Kept for tests and the bounded variants' docs; the async serve/seal paths use +// the `_bounded` forms. +#[allow(dead_code)] pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -305,6 +308,7 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> } /// Read an object's content if its type is already known. +#[allow(dead_code)] pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") .args(["cat-file", obj_type, sha256_hex]) @@ -737,6 +741,7 @@ pub fn read_object_bounded( /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// /// Returns `None` if the object does not exist in this repo. +#[allow(dead_code)] pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result)>> { let obj_type = match object_type(repo_path, sha256_hex)? { Some(t) => t, diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 086669947..4277314b5 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -12,6 +12,18 @@ use std::process::Stdio; use std::sync::mpsc; use std::time::{Duration, Instant}; +/// A (oid, path) pair for a git object reachable in the repo walk. +type ObjectPath = (String, String); + +/// Four sets derived from one walk: allowed blobs, allowed trees, all blob OIDs, +/// all tree OIDs. +type BlobTreeSets = ( + HashSet, + HashSet, + HashSet, + HashSet, +); + /// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). /// The walk is fast for a real repo; this bound exists to reap a hung or /// pathologically slow git child so it cannot pin a served-git permit (the read @@ -494,6 +506,122 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result`. +/// Used to derive both allowed blobs and allowed trees from a single walk, so +/// the two sets are consistent and the walk cost is paid only once. +fn all_object_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result<(Vec, Vec)> { + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; + + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); + let mut blob_set: HashSet<(String, String)> = HashSet::new(); + let mut tree_set: HashSet<(String, String)> = HashSet::new(); + // Phase 1: enumerate objects from ls-tree per commit (gives paths). + for commit in commits_stdout.lines() { + let commit = commit.trim(); + if commit.is_empty() { + continue; + } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rz {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + match kind { + Some("blob") => { + if let Some(oid) = oid { + blob_set.insert((oid.to_string(), format!("/{path}"))); + } + } + Some("tree") => { + if let Some(oid) = oid { + tree_set.insert((oid.to_string(), format!("/{path}"))); + } + } + _ => {} + } + } + } + // Phase 2: enumerate ALL reachable objects via cat-file --batch-all-objects. + // This catches dangling objects and objects reachable only through non-commit + // refs (tags, notes) that ls-tree misses. Objects found only here have no + // path, so they are inserted into the OID sets without a path. The allow + // filter in allowed_blob_tree_sets_bounded explicitly denies empty-path + // entries (unknown provenance), ensuring they never reach a public pin backend. + let batch_out = run_bounded_git( + git_bin, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname) %(objecttype)", + ], + repo_path, + b"", + deadline, + )?; + let batch_stdout = String::from_utf8_lossy(&batch_out); + for line in batch_stdout.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let mut parts = line.split_whitespace(); + let oid = match parts.next() { + Some(o) => o, + None => continue, + }; + let kind = parts.next(); + match kind { + // Only insert if not already present (ls-tree gives path, this + // catch-all has no path; prefer the path-annotated entry). + Some("blob") if !blob_set.iter().any(|(o, _)| o == oid) => { + blob_set.insert((oid.to_string(), String::new())); + } + Some("tree") if !tree_set.iter().any(|(o, _)| o == oid) => { + tree_set.insert((oid.to_string(), String::new())); + } + _ => {} + } + } + Ok(( + blob_set.into_iter().collect(), + tree_set.into_iter().collect(), + )) +} + /// Blob OIDs the caller may not read. A blob is withheld only if visibility /// denies the caller at *every* path the blob appears at; a blob that is also /// reachable through an allowed path is sent (its content is public elsewhere). @@ -612,21 +740,6 @@ pub fn replicable_blob_set( allowed_blob_set_for_caller(repo_path, rules, is_public, owner_did, None) } -/// [`replicable_blob_set`] with an injectable `git_bin` and walk `timeout`, for the -/// fail-closed full-scan pin path on the receive-pack side. -pub fn replicable_blob_set_bounded( - repo_path: &Path, - git_bin: &str, - timeout: Duration, - rules: &[VisibilityRule], - is_public: bool, - owner_did: &str, -) -> Result> { - allowed_blob_set_for_caller_bounded( - repo_path, git_bin, timeout, rules, is_public, owner_did, None, - ) -} - /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -1150,29 +1263,84 @@ pub fn reachable_commit_tag_oids_bounded( Ok(set) } -/// Objects safe to replicate, failing closed on blobs (#99). A candidate -/// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are -/// structural, never content-withheld) OR it is in `allowed_blobs` (reachable -/// and visibility-allowed). This drops both withheld reachable blobs and -/// dangling/unreachable blobs the reachable walk never classified, without -/// tagging the candidate list with per-object types. Used on the full-scan pin -/// path, where the candidate set can contain dangling objects the reachable-only -/// withheld set cannot cover; the delta path keeps `replicable_objects`. +/// Both the allowed blob set and the allowed tree set, derived from ONE bounded +/// walk so the two are consistent and the walk cost is paid only once. Returns +/// `(allowed_blobs, allowed_trees, all_blob_oids, all_tree_oids)`. +/// +/// A blob or tree is "allowed" if visibility permits it at *some* reachable +/// path; a tree reachable at both an allowed and denied path is allowed (its +/// metadata is public elsewhere). Commits and tags are not classified here — +/// the caller decides per type whether the allow-set applies. +pub fn allowed_blob_tree_sets_bounded( + repo_path: &Path, + git_bin: &str, + deadline: Instant, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, +) -> Result { + let (blob_pairs, tree_pairs) = all_object_paths(repo_path, git_bin, deadline)?; + let all_blob_oids: HashSet = blob_pairs.iter().map(|(oid, _)| oid.clone()).collect(); + let all_tree_oids: HashSet = tree_pairs.iter().map(|(oid, _)| oid.clone()).collect(); + let mut allowed_blobs = HashSet::new(); + for (oid, path) in &blob_pairs { + // Empty path means unknown provenance (cat-file catch-all with no + // ls-tree match). Deny rather than letting it fall through to the + // repo-wide default — an unclassified object must not enter a public + // pin backend. + if !path.is_empty() + && visibility_check(rules, is_public, owner_did, None, path) == Decision::Allow + { + allowed_blobs.insert(oid.clone()); + } + } + let mut allowed_trees = HashSet::new(); + for (oid, path) in &tree_pairs { + if !path.is_empty() + && visibility_check(rules, is_public, owner_did, None, path) == Decision::Allow + { + allowed_trees.insert(oid.clone()); + } + } + Ok((allowed_blobs, allowed_trees, all_blob_oids, all_tree_oids)) +} + +/// Objects safe to replicate, failing closed on blobs (#99) and denied trees +/// (#172). A candidate replicates iff: +/// - it is a commit (structural metadata, always safe), OR +/// - it is a blob AND is in `allowed_blobs` (reachable and visibility-allowed), OR +/// - it is a tree AND is in `allowed_trees` (reachable and visibility-allowed). +/// +/// This drops withheld blobs, withheld trees, and dangling/unreachable objects. +/// Used on the full-scan pin path, where the candidate set can contain objects +/// the reachable-only withheld set cannot cover; the delta path keeps +/// `replicable_objects`. pub fn replicable_objects_fail_closed( candidates: Vec, allowed_blobs: &HashSet, all_blob_oids: &HashSet, + allowed_trees: &HashSet, + all_tree_oids: &HashSet, ) -> Vec { candidates .into_iter() - .filter(|oid| !all_blob_oids.contains(oid) || allowed_blobs.contains(oid)) + .filter(|oid| { + if all_blob_oids.contains(oid) { + // Blobs: fail closed — only allowed blobs pass. + allowed_blobs.contains(oid) + } else if all_tree_oids.contains(oid) { + // Trees: fail closed — only allowed trees pass (#172). + // A denied tree exposes child filenames and blob OIDs even + // though the secret content itself is excluded. + allowed_trees.contains(oid) + } else { + // Commits/tags: structural metadata, always safe. + true + } + }) .collect() } -/// For every blob withheld from anonymous, the DIDs allowed to read it: the -/// owner plus any reader DID that `visibility_check` Allows at some path the -/// blob appears at. Least-privilege: a reader of one private subtree is not a -/// recipient of a blob that only lives in another. #[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, @@ -2393,6 +2561,8 @@ esac\n"; .into_iter() .map(String::from) .collect(); + let allowed_trees: HashSet = HashSet::new(); + let all_trees: HashSet = HashSet::new(); let candidates = vec![ "commit1".to_string(), "tree1".to_string(), @@ -2400,7 +2570,13 @@ esac\n"; "b_secret".to_string(), "b_dangling".to_string(), ]; - let got = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let got = replicable_objects_fail_closed( + candidates, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + ); assert_eq!( got, vec![ @@ -2483,7 +2659,15 @@ esac\n"; // Full-scan candidate set includes the dangling blob; fail-closed drops it. let candidates = vec![dangling_oid.clone(), public_oid.clone()]; - let replicable = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let allowed_trees: HashSet = HashSet::new(); + let all_trees: HashSet = HashSet::new(); + let replicable = replicable_objects_fail_closed( + candidates, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + ); assert!( !replicable.contains(&dangling_oid), "#99: a dangling private blob must not replicate" diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3b..764fca8fb 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -1434,6 +1434,59 @@ fn note_legacy_repair_read() { /// have to be documented, validated, and kept meaningful. pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); +/// A captured per-repo visibility-policy epoch that fences a pin batch. +/// +/// The reconciliation sweep reads the epoch immediately before dispatching a +/// pin loop and passes a fence in; the loop re-reads the epoch before every +/// upload and aborts the batch the moment it moves. A visibility narrow that +/// lands mid-batch (a rule made private, a repo quarantined) must not let the +/// remaining pre-authorized objects still go to a public content-addressed +/// backend — the narrow is a policy change, and dispatching against the stale +/// snapshot is the exact irreversible-publication class this fence exists for +/// (R1-P1). `None` (the push path) means "no fence": the push derives its own +/// object list at admission and holds a write lease, so no sweep-style batch +/// snapshot crosses the dispatch boundary. +#[derive(Clone)] +pub struct PolicyFence { + db: crate::db::Db, + repo_id: String, + epoch: i64, +} + +impl PolicyFence { + /// Capture the current policy epoch for `repo_id`. A read failure is a + /// skip, not a retry-with-zero: the caller must not dispatch a batch it + /// cannot fence (fail closed on a stale allow). + pub async fn capture(db: &crate::db::Db, repo_id: &str) -> Option { + match db.repo_policy_epoch(repo_id).await { + Ok(epoch) => Some(PolicyFence { + db: db.clone(), + repo_id: repo_id.to_string(), + epoch, + }), + Err(e) => { + tracing::warn!(repo = %repo_id, err = %e, "policy-epoch read failed; not fencing pin batch"); + None + } + } + } + + /// Whether the repo's policy epoch is unchanged since capture. A read + /// failure is treated as "changed": never dispatch on a policy we cannot + /// prove current. + pub async fn is_current(&self) -> bool { + match self.db.repo_policy_epoch(&self.repo_id).await { + Ok(epoch) => epoch == self.epoch, + Err(_) => false, + } + } + + /// The repo this fence guards, for log correlation. + pub fn repo_id(&self) -> &str { + &self.repo_id + } +} + /// The smallest remainder worth starting a bounded git read (or an add) with. /// /// A 1ms remainder otherwise buys a child spawned already past its deadline, which @@ -1530,6 +1583,15 @@ pub async fn pin_git_object( // Kubo returns newline-delimited JSON; we only care about the last object // (there's typically just one for a single-file add). + // + // The response MUST carry a real `Hash`: a misconfigured `GITLAWB_IPFS_API` + // (proxy returning HTML, health check on the wrong port, truncated gateway) + // can otherwise answer 2xx with no JSON, and falling back to the locally + // computed `expected_cid` would record a row for bytes the backend never + // stored. The reconciliation sweep trusts `pinned_cids` rows as durability + // evidence, so a silent false positive at pin time becomes a permanent blind + // spot for the backstop. A missing `Hash` fails the pin rather than recording + // a phantom row (mirrors Pinata's `data.cid` check). let body = resp .text() .await @@ -1541,8 +1603,26 @@ pub async fn pin_git_object( let v: serde_json::Value = serde_json::from_str(line).ok()?; v["Hash"].as_str().map(|s| s.to_string()) }) - .next_back() - .unwrap_or(expected_cid.clone()); + .next_back(); + let cid = match cid { + Some(cid) => { + if cid != expected_cid { + tracing::warn!( + sha256 = %sha256_hex, + returned = %cid, + expected = %expected_cid, + "IPFS returned a different CID than computed locally (Kubo chunking may differ); recording the backend's answer" + ); + } + cid + } + None => { + return Err(anyhow::anyhow!( + "IPFS /api/v0/add returned 2xx without a Hash field; refusing to record \ + a CID the backend never acknowledged (misconfigured GITLAWB_IPFS_API?)" + )); + } + }; tracing::debug!(sha256 = %sha256_hex, %cid, "pinned git object to IPFS"); Ok(cid) @@ -1620,8 +1700,8 @@ pub(crate) fn batch_budget_gate( /// than [`PIN_READ_FLOOR`] left. It is a gate, not a hard ceiling, since a started /// iteration still runs to completion; /// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the -/// ABSOLUTE batch deadline (not the loop-top remainder, which the `is_pinned` round-trip -/// sitting between the two would push past it), with SIGTERM-then-SIGKILL +/// ABSOLUTE batch deadline (not the loop-top remainder, which the `has_ipfs_cid` +/// round-trip sitting between the two would push past it), with SIGTERM-then-SIGKILL /// process-group teardown, so a hung `git cat-file` costs this batch its remaining /// budget plus one watchdog teardown instead of holding the permit for the child's /// whole lifetime and blocking a runtime worker while it does; @@ -1665,10 +1745,11 @@ pub(crate) fn batch_budget_gate( /// # Truncation semantics /// /// A batch stopped at the deadline leaves its remaining objects unpinned, and -/// nothing sweeps them up afterwards. There is no reconciliation pass over -/// `pinned_cids`; recovery is opportunistic, happening only if some later push -/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and -/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// nothing sweeps them up afterwards on the push path; recovery is opportunistic +/// (a later full-scan push re-offers the skipped OIDs). The reconciliation +/// sweep is the systematic backstop: when it is enabled and a pin backend is +/// configured, it re-derives the public object set each pass and fills any +/// remaining gap. /// /// The twin in `pinata.rs` is back at parity on everything that bounds or repairs an /// object: it runs the same shared budget gate at the top of every iteration, the same @@ -1699,6 +1780,7 @@ pub async fn pin_new_objects( db: &crate::db::Db, repo_id: &str, batch_budget: Duration, + fence: Option<&PolicyFence>, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -1709,6 +1791,19 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for (attempted, sha) in object_list.into_iter().enumerate() { + // Policy fence (R1-P1): a visibility narrow that lands after the caller + // built this batch must abort it before the next irreversible upload. + // Checked FIRST so a changed policy costs nothing beyond the read. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id, + unattempted = total - attempted, + "visibility policy changed mid-batch; stopping the pin loop" + ); + break; + } + } // Top of the iteration, before any of this object's work: an object is // never started with a remainder too small to cover a bounded read's // teardown. Consumed as a guard only: the read below runs against the @@ -1839,7 +1934,7 @@ pub async fn pin_new_objects( } Ok(false) => {} Err(e) => { - tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); + tracing::warn!(sha = %sha, err = %e, "DB error checking IPFS pinned status"); continue; } } @@ -1853,7 +1948,7 @@ pub async fn pin_new_objects( // own deadline regardless. // // The read runs against the ABSOLUTE batch deadline, not against the remainder - // measured at the top of the iteration: the `is_pinned` round-trip above sits + // measured at the top of the iteration: the `has_ipfs_cid` round-trip above sits // between the two, so `Instant::now() + budget_left` would land past `deadline` // by however long the DB took, and under a saturated pool that is the dominant // term. A slow DB check must not push the read's own bound out. @@ -1938,6 +2033,24 @@ pub async fn pin_new_objects( break; }; + // Dispatch fence (R1-P1): re-read the policy epoch immediately before + // the irreversible HTTP POST. The iteration-top check catches a narrow + // that landed before work began; THIS check catches a narrow that landed + // during the has_ipfs_cid round-trip or the bounded Git read — both of + // which can take seconds and during which a quarantine or rule change may + // have committed. Without this, stale plaintext can start uploading + // under authorization that is no longer current. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id, + unattempted = total - attempted, + "visibility policy changed during preparation; aborting IPFS upload" + ); + break; + } + } + // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data, Some(add_timeout)).await { Ok(cid) if !cid.is_empty() => { @@ -2200,7 +2313,7 @@ mod tests { endpoint } - /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// A sleeping-but-live endpoint. Answers `200` with a JSON `Hash` after /// `delays[i]` for the i-th request it accepts (the last entry repeats), so /// a test can make one add slow and the next fast. Drains the full request, /// headers plus the declared `Content-Length` body, before sleeping: exactly @@ -2208,8 +2321,9 @@ mod tests { /// a write failure on the client and turn a slow-but-healthy add into a /// different failure shape. /// - /// An empty body is a successful pin: `pin_git_object` falls back to the CID - /// it computed from the bytes when the response carries no `Hash`. + /// The response carries a real `Hash` because `pin_git_object` now refuses + /// to record a CID a 2xx body did not actually acknowledge: a successful + /// pin needs `{"Hash":"..."}`, not an empty body. async fn delaying_endpoint(delays: Vec) -> String { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -2246,9 +2360,14 @@ mod tests { } } tokio::time::sleep(delay).await; + let body = b"{\"Hash\":\"QmDelayMockCid\"}"; let _ = sock - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + .as_bytes(), + ) .await; + let _ = sock.write_all(body).await; let _ = sock.flush().await; }); } @@ -2335,6 +2454,74 @@ mod tests { ); } + /// The misconfigured-`GITLAWB_IPFS_API` false positive (P3): a 2xx response + /// that carries no `Hash` field (proxy returning HTML, health check on the + /// wrong port, truncated gateway) must FAIL the pin, not fall back to the + /// locally computed `expected_cid`. Falling back records a `pinned_cids` + /// row for bytes the backend never stored, and the reconciliation sweep + /// trusts rows as durability evidence — so the false positive becomes a + /// permanent blind spot for the backstop. A missing `Hash` must surface as + /// an explicit error, never a successful pin. + #[tokio::test] + async fn pin_git_object_rejects_a_2xx_without_a_hash_field() { + let endpoint = empty_ok_endpoint().await; + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object(&endpoint, "deadbeef", b"some object bytes\n", None), + ) + .await + .expect("wedge guard: an immediate empty 200 cannot take 30s"); + let err = inner.expect_err( + "a 2xx without a Hash field must not surface as a successful pin \ + (would record a phantom pinned_cids row the sweep then trusts)", + ); + assert!( + err.to_string().contains("without a Hash field"), + "the error must name the missing Hash so operators diagnose the endpoint: {err:#}" + ); + } + + /// A 200 that answers with an empty body and no `Hash` — the exact shape of + /// a proxy or health-check endpoint mistaken for a Kubo API. + async fn empty_ok_endpoint() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. #[tokio::test] async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { @@ -2382,6 +2569,7 @@ mod tests { &db, "repo-batch-budget", Duration::from_millis(5500), + None, ), ) .await @@ -2449,6 +2637,7 @@ mod tests { &db, "repo-batch-continues", Duration::from_secs(90), + None, ), ) .await @@ -2487,6 +2676,7 @@ mod tests { &db, "repo-batch-rejects", Duration::from_secs(60), + None, ), ) .await @@ -2584,6 +2774,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(2), + None, ), ) .await @@ -2671,6 +2862,7 @@ mod tests { &db, "repo-merge-test", Duration::from_millis(1500), + None, ), ) .await @@ -2744,6 +2936,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -2818,6 +3011,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -2886,6 +3080,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 66bfa0961..f2125cf68 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -16,6 +16,7 @@ mod operator; mod p2p; mod pinata; mod rate_limit; +mod reconciliation; mod server; mod state; mod sync; @@ -642,6 +643,29 @@ async fn main() -> Result<()> { info!("auto-sync worker started"); } + // Periodic reconciliation sweep: re-derives pin/seal sets and fills gaps + // so a dropped replication job never means data loss. + { + let db = state.db.clone(); + let config = Arc::clone(&state.config); + let http_client = Arc::clone(&state.http_client); + let node_keypair = Arc::clone(&state.node_keypair); + let node_did = state.node_did.clone(); + let pin_sem = Arc::clone(&state.pin_semaphore); + let shutdown_rx = state.subscribe_shutdown(); + if reconciliation::spawn( + db, + config, + http_client, + node_keypair, + node_did, + pin_sem, + shutdown_rx, + ) { + info!("reconciliation sweep worker started"); + } + } + // On-chain operator setup: verify stake + spawn heartbeat loop if !state.config.contract_node_staking.is_empty() && !state.config.operator_private_key.is_empty() diff --git a/crates/gitlawb-node/src/metrics.rs b/crates/gitlawb-node/src/metrics.rs index c95ef1d18..85c98f488 100644 --- a/crates/gitlawb-node/src/metrics.rs +++ b/crates/gitlawb-node/src/metrics.rs @@ -15,6 +15,9 @@ //! `gitlawb_pack_size_bytes` //! * a single `gitlawb_info{version, did}` gauge = 1, for joins/dashboards //! * currently-connected peer count — `gitlawb_peers_connected` +//! * reconciliation sweep gaps found and filled — +//! `gitlawb_reconciliation_gaps_found_total` / +//! `gitlawb_reconciliation_gaps_filled_total` //! //! All metrics live in a single process-wide registry initialized by //! [`init`]. Increment helpers (`record_push`, `record_auth_failure`, ...) @@ -33,8 +36,8 @@ use std::sync::OnceLock; use prometheus::{ - Encoder, Histogram, HistogramOpts, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, - TextEncoder, + Encoder, Histogram, HistogramOpts, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts, + Registry, TextEncoder, }; /// The single, process-wide metrics registry. Initialized by [`init`]. @@ -51,6 +54,8 @@ static SYNC_PROCESSED: OnceLock = OnceLock::new(); static WEBHOOK_DELIVERIES: OnceLock = OnceLock::new(); static PACK_SIZE: OnceLock = OnceLock::new(); static PEERS_CONNECTED: OnceLock = OnceLock::new(); +static RECONCILIATION_GAPS_FOUND: OnceLock = OnceLock::new(); +static RECONCILIATION_GAPS_FILLED: OnceLock = OnceLock::new(); /// One-time initializer. Builds the registry, registers every metric, /// and sets the constant `gitlawb_info` gauge. Idempotent — calling @@ -202,6 +207,30 @@ fn init_inner(version: &str, node_did: &str) { .set(peers_connected) .expect("set PEERS_CONNECTED once"); + let gaps_found = IntCounter::with_opts(Opts::new( + "gitlawb_reconciliation_gaps_found_total", + "Total reconciliation sweep gaps detected (objects that should be pinned but are not)", + )) + .expect("gitlawb_reconciliation_gaps_found_total definition"); + registry + .register(Box::new(gaps_found.clone())) + .expect("register gitlawb_reconciliation_gaps_found_total"); + RECONCILIATION_GAPS_FOUND + .set(gaps_found) + .expect("set RECONCILIATION_GAPS_FOUND once"); + + let gaps_filled = IntCounter::with_opts(Opts::new( + "gitlawb_reconciliation_gaps_filled_total", + "Total reconciliation sweep gaps successfully filled (objects pinned by the sweep)", + )) + .expect("gitlawb_reconciliation_gaps_filled_total definition"); + registry + .register(Box::new(gaps_filled.clone())) + .expect("register gitlawb_reconciliation_gaps_filled_total"); + RECONCILIATION_GAPS_FILLED + .set(gaps_filled) + .expect("set RECONCILIATION_GAPS_FILLED once"); + REGISTRY .set(registry) .expect("set REGISTRY once (init must be called exactly once)"); @@ -284,6 +313,20 @@ pub fn set_peers_connected(count: i64) { } } +/// Record reconciliation sweep gaps found (objects that should be pinned but are not). +pub fn record_reconciliation_gaps_found(count: u64) { + if let Some(c) = RECONCILIATION_GAPS_FOUND.get() { + c.inc_by(count); + } +} + +/// Record reconciliation sweep gaps filled (objects successfully pinned by the sweep). +pub fn record_reconciliation_gaps_filled(count: u64) { + if let Some(c) = RECONCILIATION_GAPS_FILLED.get() { + c.inc_by(count); + } +} + /// Encode the registry as the standard Prometheus text exposition format. /// Returns an error if `init` was never called. pub fn encode() -> Result { @@ -321,6 +364,8 @@ mod tests { .expect("PUSHES set after init") .with_label_values(&["alice/repo"]) .inc(); + record_reconciliation_gaps_found(7); + record_reconciliation_gaps_filled(3); let body = encode().expect("encode should succeed after init"); assert!( @@ -335,6 +380,14 @@ mod tests { body.contains("gitlawb_pushes_total{repo=\"alice/repo\"} 1"), "expected the incremented counter to be visible in: {body}" ); + assert!( + body.contains("gitlawb_reconciliation_gaps_found_total 7"), + "expected the reconciliation gaps-found counter to be visible in: {body}" + ); + assert!( + body.contains("gitlawb_reconciliation_gaps_filled_total 3"), + "expected the reconciliation gaps-filled counter to be visible in: {body}" + ); } /// #192 F4: `init` is idempotent and safe to call repeatedly. The panic that diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 14f1d5824..3ecc64388 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -169,6 +169,7 @@ pub async fn pin_new_objects( db: &crate::db::Db, repo_id: &str, batch_budget: Duration, + fence: Option<&crate::ipfs_pin::PolicyFence>, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -179,6 +180,18 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for (attempted, sha) in object_list.into_iter().enumerate() { + // Policy fence (R1-P1): a visibility narrow that lands after the caller + // built this batch must abort it before the next irreversible upload. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + unattempted = total - attempted, + "visibility policy changed mid-batch; stopping the Pinata pin loop" + ); + break; + } + } // Top of the iteration, before any of this object's work: an object is never // started with a remainder too small to cover a bounded read's teardown. The // gate is shared with the IPFS loop so the two cannot drift apart in how they @@ -394,6 +407,21 @@ pub async fn pin_new_objects( } }; + // Dispatch fence (R1-P1): re-read the policy epoch immediately before + // the irreversible HTTP POST. The iteration-top check catches a narrow + // that landed before work began; THIS check catches a narrow that landed + // during the has_pinata_cid round-trip or the bounded Git read. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + unattempted = total - attempted, + "visibility policy changed during preparation; aborting Pinata upload" + ); + break; + } + } + match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { // The resolver key (`pinned_cids.cid`) must be the locally-computed @@ -687,6 +715,7 @@ mod tests { &db, "repo-merge-test", Duration::from_millis(5500), + None, ), ) .await @@ -778,6 +807,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(2), + None, ), ) .await @@ -968,6 +998,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1042,6 +1073,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1093,6 +1125,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1122,6 +1155,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1175,6 +1209,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs new file mode 100644 index 000000000..20ea86030 --- /dev/null +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -0,0 +1,1703 @@ +use rand::Rng; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::watch; + +use crate::config::Config; +use crate::db::Db; + +/// How often to run a sweep pass. +const SWEEP_INTERVAL_SECS: u64 = 3600; + +/// Maximum repos to process per pass — prevents the sweep from becoming +/// the O(repos) amplification the admission-control work exists to prevent. +const REPOS_PER_PASS: usize = 100; + +/// Maximum objects to pin per backend per repo in a single pass — prevents one +/// large repo from monopolizing the blocking pool or the hourly budget. Applied +/// after filtering out already-pinned objects so the cap reflects actual work. +const MAX_OBJECTS_PER_REPO: usize = 50_000; + +/// Per-repo deadline for the blocking git scan (list_all_objects + visibility +/// filter). A pathological repo that stalls past this is skipped for the pass. +const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); + +/// Per-repo deadline for the pinning phase (IPFS + Pinata uploads). An +/// unavailable backend that stalls per-object must not hold the sweep for +/// the entire backlog; this bounds the wall time of each pinning PHASE. +/// +/// The phases do NOT share one budget (R2-P3): the scan, the mid-scan +/// visibility re-filter, the per-backend pin-boundary authorization +/// re-derivation, the withheld-blob walk, and each pin/seal phase each get +/// their own `REPO_SCAN_DEADLINE` / `PIN_PHASE_DEADLINE`. A repo's worst case +/// is therefore ADDITIVE, up to ~30min in pathological conditions (scan 5m + +/// mid-scan re-filter 5m + authz re-derivation 5m + withheld walk 5m + public +/// pin 5m + encrypted seal 5m), not bounded at a single deadline. That is a +/// deliberate trade: starving a later phase of the budget the scan consumed +/// would silently disable the authorization check or the recovery-copy seal +/// for exactly the large repos the sweep exists for. The sweep runs hourly +/// and each phase is still individually bounded, so a pathological repo delays +/// other repos by at most that phase, not the hour. +const PIN_PHASE_DEADLINE: Duration = Duration::from_secs(300); + +/// node_state key under which the sweep's keyset cursor is persisted across +/// restarts (R2-P1). +const CURSOR_KEY: &str = "reconciliation_sweep_cursor"; + +/// Whether the sweep should spawn given the current configuration. +/// Extracted for testing — test both directions independently. +fn should_spawn(config: &Config) -> bool { + if !config.reconciliation_sweep { + return false; + } + !config.ipfs_api.is_empty() || !config.pinata_jwt.is_empty() +} + +/// Spawn the periodic reconciliation sweep background task. +/// No-op when neither IPFS nor Pinata is configured, or when +/// `reconciliation_sweep` is disabled. Returns `true` when the worker was +/// actually spawned so the caller can gate its own "worker started" logging. +pub fn spawn( + db: Arc, + config: Arc, + http_client: Arc, + node_keypair: Arc, + node_did: gitlawb_core::did::Did, + pin_sem: Arc, + mut shutdown_rx: watch::Receiver, +) -> bool { + if !should_spawn(&config) { + tracing::info!( + "reconciliation sweep: disabled or neither IPFS nor Pinata configured, skipping spawn" + ); + return false; + } + + tokio::spawn(async move { + let node_seed = *node_keypair.to_seed(); + // Resume from the persisted cursor (R2-P1): a node restart must not + // re-walk every repo, and the cursor is only ever advanced after a + // batch completes, so an interrupted pass resumes where it stopped. + let mut cursor: Option = match db.get_node_state(CURSOR_KEY).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(err = %e, "failed to load reconciliation sweep cursor from node_state; starting from scratch"); + None + } + }; + + // First pass: random delay to desynchronize sweep starts across nodes + // on a rolling restart (R1-P3). Subsequent passes use the fixed interval. + // Generate the delay before the async block to avoid Send issues with thread_rng. + let initial_delay = Duration::from_millis(rand::thread_rng().gen_range(0..60000)); + let mut first_pass = true; + + loop { + // On first pass, wait for the initial random delay before starting + if first_pass { + tracing::debug!( + delay_ms = initial_delay.as_millis() as u64, + "reconciliation sweep: waiting initial jitter delay" + ); + tokio::select! { + _ = tokio::time::sleep(initial_delay) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received during initial delay, exiting"); + return; + } + } + } + first_pass = false; + } + + let start = std::time::Instant::now(); + match run_pass( + &db, + &config, + &http_client, + &node_seed, + &node_did, + &pin_sem, + REPO_SCAN_DEADLINE, + &mut cursor, + &mut shutdown_rx, + ) + .await + { + Ok((count, gaps, filled)) => { + tracing::info!( + repos = count, + gaps_found = gaps, + gaps_filled = filled, + elapsed_ms = start.elapsed().as_millis() as u64, + "reconciliation sweep pass complete" + ); + } + Err(e) => { + tracing::warn!(err = %e, "reconciliation sweep pass failed"); + } + } + + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received, exiting"); + return; + } + + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(SWEEP_INTERVAL_SECS)) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received, exiting"); + return; + } + } + } + } + }); + + true +} + +/// Re-derive the *allowed* public-object set from fresh rules and intersect it +/// with the scanned object list. Returns `None` when the re-derivation failed +/// (caller skips the repo). This is the path-scoped-visibility re-filter that +/// runs against rules re-fetched after the git scan, so a narrowing made +/// mid-scan is honored before anything is pinned. +/// +/// The caller hands an absolute `deadline`; the whole re-derivation +/// (replicable_blob_set_bounded + all_blob_oids) runs against the remaining +/// budget rather than granting each git child a fresh timeout. The mid-scan +/// re-filter and each pin-boundary re-derivation each get their OWN fresh +/// `REPO_SCAN_DEADLINE` (R2-P1) so a scan that exhausts its own budget cannot +/// disable the authorization-at-dispatch recheck — the read phase is additive +/// with the pin phases, documented at `PIN_PHASE_DEADLINE`. +async fn refilter_public_objects( + disk: &std::path::Path, + rules: &[crate::db::VisibilityRule], + is_public: bool, + owner_did: &str, + object_list: Vec, + deadline: Instant, +) -> Option> { + let disk_clone = disk.to_path_buf(); + let rules_clone = rules.to_vec(); + let owner_clone = owner_did.to_string(); + + match tokio::time::timeout( + deadline.saturating_duration_since(Instant::now()), + tokio::task::spawn_blocking(move || -> anyhow::Result> { + // The shared deadline spans this whole re-filter + // (allowed_blob_tree_sets_bounded), so a slow walk is bounded as a + // unit rather than granting each git child a fresh timeout. + let (allowed, allowed_trees, all_blobs, all_trees) = + crate::git::visibility_pack::allowed_blob_tree_sets_bounded( + &disk_clone, + "git", + deadline, + &rules_clone, + is_public, + &owner_clone, + )?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + object_list, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + )) + }), + ) + .await + { + Ok(Ok(Ok(list))) => Some(list), + Ok(Ok(Err(e))) => { + tracing::warn!(err = %e, "visibility re-derivation failed"); + None + } + Ok(Err(e)) => { + tracing::warn!(err = %e, "visibility re-derivation task panicked"); + None + } + Err(_) => { + tracing::warn!("visibility re-derivation deadline exceeded"); + None + } + } +} +/// Re-check quarantine AND root visibility immediately before an irreversible +/// public pin (R1-P1). Returns the fresh repo row plus fresh rules, or `None` +/// when the pin must be skipped. DB failures are treated as skip (never pin on +/// a stale allow), so one repo's failure does not abort the pass. +async fn recheck_public_pin( + db: &Db, + repo_id: &str, + repo_slug: &str, +) -> Option<(crate::db::RepoRecord, Vec)> { + match db.is_repo_quarantined(repo_id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping pin"); + return None; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine recheck failed, skipping pin"); + return None; + } + } + let rules = match db.list_visibility_rules(repo_id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed, skipping pin"); + return None; + } + }; + let fresh = match db.get_repo_by_id(repo_id).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!(repo = %repo_slug, "repo disappeared from DB, skipping pin"); + return None; + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed, skipping pin"); + return None; + } + }; + if !crate::visibility::listable_at_root(&rules, fresh.is_public, &fresh.owner_did, None) { + tracing::warn!(repo = %repo_slug, "visibility narrowed, skipping pin"); + return None; + } + Some((fresh, rules)) +} + +/// Compute the deterministic missing set: `all` minus `done`, sorted so two +/// passes over the same data yield the same pin order. Not capped here — the +/// caller applies the cap and logs a truncation warning. +fn missing_oids(all: &[String], done: &[String]) -> Vec { + let done_set: HashSet<&str> = done.iter().map(|s| s.as_str()).collect(); + let mut missing: Vec = all + .iter() + .filter(|s| !done_set.contains(s.as_str())) + .cloned() + .collect(); + missing.sort(); + missing +} + +/// Cap a missing set, logging once when it was truncated. +fn cap_missing(v: Vec, repo_slug: &str, backend: &str) -> Vec { + if v.len() > MAX_OBJECTS_PER_REPO { + tracing::warn!( + repo = %repo_slug, + backend, + cap = MAX_OBJECTS_PER_REPO, + "per-repo missing cap reached, truncating" + ); + let mut v = v; + v.truncate(MAX_OBJECTS_PER_REPO); + v + } else { + v + } +} + +/// Run one sweep pass. Returns `(repos_scanned, gaps_found, gaps_filled)`. +/// +/// `repos_scanned` counts every repo actually visited this pass (mirror rows +/// and hard skips excluded, and the loop stops counting the moment a shutdown +/// signal breaks the batch), so the returned value never overreports work that +/// a mid-pass shutdown prevented (R1-P3). +/// +/// Nine args but grouping them would churn every test caller for no behavioral +/// gain; the pins each arg names are independently documented at their use. +/// `rederive_budget` is the budget each authorization-at-dispatch +/// re-derivation runs against: the mid-scan re-filter and each pin-boundary +/// re-derivation compute their OWN fresh `Instant::now() + rederive_budget` +/// (R2-P1), so a scan that exhausts `REPO_SCAN_DEADLINE` cannot starve the +/// visibility recheck that runs right before anything is pinned. Plumbed +/// through the signature (rather than read as a module const) so the call-site +/// wiring is testable. +#[allow(clippy::too_many_arguments)] +async fn run_pass( + db: &Db, + config: &Config, + http_client: &reqwest::Client, + node_seed: &[u8; 32], + node_did: &gitlawb_core::did::Did, + pin_sem: &Arc, + rederive_budget: Duration, + cursor: &mut Option, + shutdown_rx: &mut watch::Receiver, +) -> anyhow::Result<(usize, usize, usize)> { + // Keyset pagination over repos ordered by immutable id so the cursor is + // robust against insertions, deletions, or updated_at shifts. The LIMIT + // is pushed into the SQL query so the hourly pass does not allocate, + // transfer, or deduplicate every repo on every sweep. + // + // Fetch one EXTRA row as a lookahead (R1-P2): `batch.len() < REPOS_PER_PASS` + // is a wrong "final page" proxy when the key space ends on an exact multiple + // of the page size — that batch LOOKS full, yet no row follows. With a + // lookahead row present, the batch is full for real (more remain); without + // it, the batch is the terminal page even at exactly REPOS_PER_PASS rows. + let fetched = db + .list_all_repos_deduped_stable(cursor.as_deref(), REPOS_PER_PASS as i64 + 1) + .await?; + let has_more = fetched.len() > REPOS_PER_PASS; + let batch: Vec<_> = fetched.into_iter().take(REPOS_PER_PASS).collect(); + + if batch.is_empty() { + // Covered everything: clear the persisted cursor so the next pass + // starts a fresh cycle instead of wedging on a stale key. + *cursor = None; + db.set_node_state(CURSOR_KEY, None).await?; + return Ok((0, 0, 0)); + } + + // Advance the in-memory cursor now so the next page in this run continues + // after this batch; the PERSISTED cursor is only moved once the batch fully + // completes below, so an interrupted batch is re-walked on restart. + let batch_last = batch.last().unwrap().id.clone(); + *cursor = Some(batch_last.clone()); + + let mut total_gaps_found = 0usize; + let mut total_gaps_filled = 0usize; + let mut repos_scanned = 0usize; + let mut batch_completed = true; + + for repo in &batch { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received mid-pass, exiting"); + batch_completed = false; + break; + } + + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&repo.owner_did), + repo.name + ); + + // Mirror rows carry a slash-form id written only by upsert_mirror_repo; + // they hardcode is_public = true and replicate no visibility rules, so a + // sweep over one would irreversibly publish content that the canonical + // gate never admitted (R2-P1). Skip them — the canonical row (if any) + // is swept under its own id. + if repo.id.contains('/') { + tracing::debug!(repo = %repo_slug, "mirror row (no canonical repo), skipping sweep"); + continue; + } + + let disk = PathBuf::from(&repo.disk_path); + if !disk.exists() { + tracing::warn!(repo = %repo_slug, "disk path missing, skipping"); + continue; + } + + // Counted only once the repo has a real chance of work: mirror rows and + // missing-disk rows are hard skips and never count as scanned (R1-P3). + repos_scanned += 1; + + // Cheap quarantine pre-check BEFORE the expensive git scan (R1-P3): + // a repo quarantined since admission should not burn a full scan just + // to be told to skip. + match db.is_repo_quarantined(&repo.id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping"); + continue; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine check failed, skipping"); + continue; + } + } + + let rules = match db.list_visibility_rules(&repo.id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules fetch failed, skipping"); + continue; + } + }; + + if !crate::visibility::listable_at_root(&rules, repo.is_public, &repo.owner_did, None) { + continue; + } + + // ── Full git scan (bounded) ───────────────────────────────────── + // One absolute deadline spans the whole scan. The mandatory visibility + // re-filter below runs against its OWN fresh budget (`authz_deadline`), + // NOT this spent deadline (R2-P1): a scan that legitimately consumes + // its whole budget would otherwise compute a zero remaining duration + // for the re-filter, time out immediately, and abort the repo + // iteration — permanently skipping exactly the large repos the sweep + // exists for. The pin-boundary re-derivations use the same fresh- + // budget pattern per backend arm, so no later authorization stage can + // be starved by the read phase's consumption. + let scan_deadline = Instant::now() + REPO_SCAN_DEADLINE; + let disk_clone = disk.clone(); + let owner_clone = repo.owner_did.clone(); + let rules_clone = rules.clone(); + let is_public = repo.is_public; + + let object_list = tokio::time::timeout( + scan_deadline.saturating_duration_since(Instant::now()), + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let all_objs = + crate::git::push_delta::list_all_objects(&disk_clone, "git", scan_deadline)?; + let (allowed, allowed_trees, all_blobs, all_trees) = + crate::git::visibility_pack::allowed_blob_tree_sets_bounded( + &disk_clone, + "git", + scan_deadline, + &rules_clone, + is_public, + &owner_clone, + )?; + // Fail closed for blobs and denied trees (#172): the + // batch-all-objects enumeration carries dangling commits/trees + // from an aborted push, which have no path scoping to fail + // closed against. Requiring membership in the reachable object + // set keeps their messages, authors, parent links, and + // tree/file-name metadata off public pin backends (R2). + let reachable = crate::git::push_delta::reachable_object_oids( + &disk_clone, + "git", + scan_deadline, + )?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + all_objs, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + ) + .into_iter() + .filter(|oid| reachable.contains(oid)) + .collect()) + }), + ) + .await; + + let object_list: Vec = match object_list { + Ok(Ok(Ok(list))) => list, + Ok(Ok(Err(e))) => { + tracing::warn!(repo = %repo_slug, err = %e, "full-scan failed, skipping"); + continue; + } + Ok(Err(e)) => { + tracing::warn!(repo = %repo_slug, err = %e, "full-scan task panicked, skipping"); + continue; + } + Err(_) => { + tracing::warn!(repo = %repo_slug, "full-scan deadline exceeded, skipping"); + continue; + } + }; + + if object_list.is_empty() { + continue; + } + + // Fresh budget for the authorization-at-dispatch re-derivations (R1/R2): + // the scan may have legitimately consumed its whole `scan_deadline`, and + // reusing that deadline here would compute a zero remaining duration, + // return None, and turn an empty `to_pin` into a permanent hourly skip + // for exactly the large/slow repos the sweep exists for. This deadline is + // deliberately NOT shared with the scan. The mid-scan re-filter and each + // backend arm each re-derive against their OWN fresh budget (R2-P1): the + // IPFS arm re-derives first, and if two stages shared one budget a large + // repo that consumed it on an earlier walk would leave the later stage + // silently skipped every pass — empty `to_pin` behind a warn. + + // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── + // Re-check quarantine AND visibility right now (fresh rules + repo row), + // then re-derive the allowed set from those fresh rules so a path-scoped + // narrowing made mid-scan is honored before anything is pinned. + let (fresh_repo, fresh_rules) = match recheck_public_pin(db, &repo.id, &repo_slug).await { + Some(v) => v, + None => continue, + }; + + // Visibility may have narrowed mid-scan with a path-scoped deny. + // Recompute the allowed set from fresh rules and intersect it with the + // existing object_list. Runs against its OWN fresh `authz_deadline`, NOT + // the spent `scan_deadline` (R2-P1): the scan may have consumed the whole + // read budget, and a reused deadline computes a zero remaining duration, + // times out immediately, and aborts the repo iteration before the pin + // phases ever run — permanently skipping exactly the large repos the + // durability backstop exists for. The pin-boundary re-derivations below + // use the same fresh-budget pattern per backend arm. + let authz_deadline = Instant::now() + rederive_budget; + let refiltered = refilter_public_objects( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + object_list, + authz_deadline, + ) + .await; + let Some(object_list) = refiltered else { + tracing::warn!(repo = %repo_slug, "fresh-visibility re-filter failed, skipping"); + continue; + }; + if object_list.is_empty() { + continue; + } + + let ipfs_enabled = !config.ipfs_api.is_empty(); + let pinata_enabled = !config.pinata_jwt.is_empty(); + + // IPFS-missing set. A filter DB error skips only the IPFS gap-fill and + // lets the Pinata path still run (R1-P3), instead of dropping the repo. + let ipfs_missing: Vec = if ipfs_enabled { + match db.filter_ipfs_pinned_oids(&object_list).await { + Ok(already) => { + cap_missing(missing_oids(&object_list, &already), &repo_slug, "IPFS") + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, IPFS gap-fill skipped this pass"); + Vec::new() + } + } + } else { + Vec::new() + }; + + let pinata_missing: Vec = if pinata_enabled { + match db.filter_pinata_pinned_oids(&object_list).await { + Ok(already) => { + cap_missing(missing_oids(&object_list, &already), &repo_slug, "Pinata") + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, Pinata gap-fill skipped this pass"); + Vec::new() + } + } + } else { + Vec::new() + }; + + // Count UNIQUE missing objects across both backends (R1-P3): an object + // absent from both must not be counted twice. + let mut gap_union: HashSet<&str> = HashSet::new(); + gap_union.extend(ipfs_missing.iter().map(|s| s.as_str())); + gap_union.extend(pinata_missing.iter().map(|s| s.as_str())); + let repo_gaps = gap_union.len(); + if repo_gaps > 0 { + total_gaps_found += repo_gaps; + crate::metrics::record_reconciliation_gaps_found(repo_gaps as u64); + } + + // Re-validate quarantine + visibility IMMEDIATELY before each backend + // pin (R1-P1) and re-derive the allowed set from the rules read at that + // moment, intersecting it with the to-pin list (R2-P1): for + // content-addressed public pins a stale allow is effectively + // irreversible, and the pin itself takes time. A path-scoped deny that + // landed after the mid-scan refilter (which only checks root listability) + // is honored here because the candidates are intersected with the set + // allowed under the fresh rules, not just root-gated. Each backend runs + // under a PolicyFence captured at ITS dispatch boundary, so a narrow that + // lands mid-batch aborts the remaining uploads (R1-P1). + // + // Acquire the same global pin permit the push path holds (R2-P2): the + // sweep's pin loops must not bypass `max_concurrent_pin_tasks`. Acquired + // only when there is actual pin work; the scan above holds no permit. + // The permit is held across the public pin loops AND the encrypted seal + // below (which also writes to IPFS) and dropped at the end of this repo's + // iteration. + let _pin_permit = if !ipfs_missing.is_empty() || !pinata_missing.is_empty() { + let permit = pin_sem.clone().acquire_owned().await?; + Some(permit) + } else { + None + }; + let ipfs_fence = if ipfs_enabled && !ipfs_missing.is_empty() { + crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await + } else { + None + }; + let pinata_fence = if pinata_enabled && !pinata_missing.is_empty() { + crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await + } else { + None + }; + + let pinned_ipfs: Vec<(String, String)> = if ipfs_enabled && !ipfs_missing.is_empty() { + match ipfs_fence { + None => { + tracing::warn!(repo = %repo_slug, "IPFS policy-epoch capture failed, skipping"); + Vec::new() + } + Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + let to_pin = match refilter_public_objects( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + ipfs_missing, + Instant::now() + rederive_budget, + ) + .await + { + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "IPFS pin-boundary re-derivation failed, skipping"); + Vec::new() + } + }; + if to_pin.is_empty() { + Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::ipfs_pin::pin_new_objects( + &config.ipfs_api, + &disk, + "git", + to_pin, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + Some(&fence), + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } + } + } + }, + } + } else { + Vec::new() + }; + + let pinned_pinata: Vec<(String, String)> = if pinata_enabled && !pinata_missing.is_empty() { + match pinata_fence { + None => { + tracing::warn!(repo = %repo_slug, "Pinata policy-epoch capture failed, skipping"); + Vec::new() + } + Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + // Own budget (R2-P1): the IPFS arm above may have + // consumed the whole shared deadline, and a reused + // spent deadline here would silently skip Pinata every + // pass for exactly the large repos this sweep exists + // for. + let to_pin = match refilter_public_objects( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + pinata_missing, + Instant::now() + rederive_budget, + ) + .await + { + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "Pinata pin-boundary re-derivation failed, skipping"); + Vec::new() + } + }; + if to_pin.is_empty() { + Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + "git", + to_pin, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + Some(&fence), + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } + } + } + }, + } + } else { + Vec::new() + }; + + // `pin_new_objects` returns only objects whose DB record was written + // (R1-P3), so a backend that uploaded bytes but failed to persist is + // not counted as "filled". Count UNIQUE objects across both backends + // (R2-P3): `gaps_found` is the union of missing OIDs, so an object + // pinned to BOTH backends must not count twice against that union. + let mut filled_union: HashSet<&String> = HashSet::new(); + filled_union.extend(pinned_ipfs.iter().map(|(sha, _)| sha)); + filled_union.extend(pinned_pinata.iter().map(|(sha, _)| sha)); + let repo_filled = filled_union.len(); + if repo_filled > 0 { + total_gaps_filled += repo_filled; + crate::metrics::record_reconciliation_gaps_filled(repo_filled as u64); + + tracing::info!( + repo = %repo_slug, + ipfs = pinned_ipfs.len(), + pinata = pinned_pinata.len(), + total = repo_filled, + "reconciliation sweep filled public-object gaps" + ); + } + + // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── + + // Fence the encrypted path from the point the recipients are derived: + // the withheld-blob walk is long, and `encrypt_and_pin` re-checks the + // epoch per blob, so a visibility rule moving mid-walk aborts the seal + // loop before a stale recipient set is pinned (R1-P1). Captured BEFORE + // the rules recheck below, mirroring the public path (R2-P1): if a rule + // change landed between a recheck-first ordering's rule read and this + // capture, the change would be baked into the recipient set while the + // epoch captured after it already reflected the move — `is_current` + // would then report current for the whole seal loop and the fence would + // never fire for that narrow. + let enc_fence = match crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await { + Some(f) => f, + None => { + tracing::warn!(repo = %repo_slug, "policy-epoch capture failed, skipping encrypted pin"); + continue; + } + }; + // Recheck quarantine AND root visibility before encrypted pinning, using + // FRESH repo identity (R1-P2): the batch snapshot may predate a narrow. + let (fresh_repo2, fresh_rules2) = match recheck_public_pin(db, &repo.id, &repo_slug).await { + Some(v) => v, + None => continue, + }; + + let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&fresh_rules2); + if has_path_scoped && ipfs_enabled { + let p = disk.clone(); + let owner = fresh_repo2.owner_did.clone(); + let r = fresh_rules2.clone(); + let is_public_2 = fresh_repo2.is_public; + let recipients = tokio::time::timeout( + REPO_SCAN_DEADLINE, + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &p, + "git", + REPO_SCAN_DEADLINE, + &r, + is_public_2, + &owner, + ) + }), + ) + .await; + + let rec = match recipients { + Ok(Ok(Ok(rec))) => rec, + Ok(Ok(Err(e))) => { + tracing::warn!( + repo = %repo_slug, err = %e, + "withheld_blob_recipients failed, skipping encrypted pin" + ); + continue; + } + Ok(Err(e)) => { + tracing::warn!( + repo = %repo_slug, err = %e, + "withheld_blob_recipients task panicked, skipping encrypted pin" + ); + continue; + } + Err(_) => { + tracing::warn!( + repo = %repo_slug, + "encrypted recovery deadline exceeded, skipping" + ); + continue; + } + }; + + if !rec.is_empty() { + // The encrypted seal writes to IPFS too, so it runs under the + // same global pin permit as the public loops (R2-P2). Reuse the + // permit `_pin_permit` already holds for this repo when the + // public phase had gaps; only acquire a fresh one when it did + // not. One permit per repo, never two (R2-P1): with + // `max_concurrent_pin_tasks = 1` a second acquire here would + // wait on the very permit this iteration holds and deadlock the + // sweep past its guard timeout. + let _enc_permit = match &_pin_permit { + Some(_) => None, + None => Some(pin_sem.clone().acquire_owned().await?), + }; + // Bound the seal+pin work (R1-P2): an unavailable backend must + // not hold the sweep past the pin-phase budget. + let sealed = tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::encrypted_pin::encrypt_and_pin( + &config.ipfs_api, + &disk, + db, + &repo.id, + node_seed, + "git", + crate::ipfs_pin::PIN_BATCH_BUDGET, + &rec, + Some(&enc_fence), + ), + ) + .await; + + let sealed: Vec<(String, String)> = match sealed { + Ok(v) => v, + Err(_) => { + tracing::warn!( + repo = %repo_slug, + "encrypted pin phase timed out after {:?}", + PIN_PHASE_DEADLINE + ); + Vec::new() + } + }; + + // Anchor only when something was newly sealed this pass. + // This avoids unbounded Irys writes on a timer — repos + // with no withheld changes do not re-anchor the manifest. + if !sealed.is_empty() && !config.irys_url.is_empty() { + // Bind the manifest to the FRESH repo identity re-fetched at + // the pin boundary (`fresh_repo2`), not the batch snapshot: + // a renamed/ownership-changed repo must not anchor encrypted + // recovery copies under a stale owner (R1-P2). + let owner_short = crate::db::normalize_owner_key(&fresh_repo2.owner_did); + let slug = format!("{}/{}", owner_short, fresh_repo2.name); + let ts = chrono::Utc::now().to_rfc3339(); + let node_did_str = node_did.to_string(); + + let manifest = crate::arweave::EncryptedManifest { + repo: &slug, + owner_did: &fresh_repo2.owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &sealed, + }; + if let Err(e) = crate::arweave::anchor_encrypted_manifest( + http_client, + &config.irys_url, + &manifest, + ) + .await + { + tracing::warn!( + repo = %slug, + err = %e, + "encrypted manifest anchor failed (will retry next pass)" + ); + } + } + } + } + } + + // Persist the cursor only when the WHOLE batch completed. If shutdown + // interrupted us, leave the persisted cursor at the previous batch's end so + // the next run re-walks the unprocessed tail (R2-P1, R1-P3). + if batch_completed { + // A terminal page (no lookahead row) means the whole key space is + // covered: clear the cursor now so the next tick starts a fresh cycle + // instead of burning one pass on an empty batch. The lookahead is what + // distinguishes "full because more remain" from "full because the key + // space ends on an exact page boundary" (R1-P2). + if !has_more { + *cursor = None; + if let Err(e) = db.set_node_state(CURSOR_KEY, None).await { + tracing::warn!(err = %e, "failed to clear reconciliation sweep cursor on final page"); + } + } else if let Err(e) = db.set_node_state(CURSOR_KEY, Some(&batch_last)).await { + tracing::warn!(err = %e, "failed to persist reconciliation sweep cursor"); + } + } + + Ok((repos_scanned, total_gaps_found, total_gaps_filled)) +} + +#[cfg(test)] +mod tests { + use tokio::sync::watch; + + /// Build a minimal Config with both IPFS and Pinata fields empty so the + /// spawn() gate fires and the function returns without touching the DB. + fn empty_pin_config() -> std::sync::Arc { + // Config derives clap::Parser; supply only argv[0] (the program name) + // so all fields get their defaults (ipfs_api = "", pinata_jwt = ""). + let cfg = ::parse_from(["gitlawb-node-test"]); + std::sync::Arc::new(cfg) + } + + /// Build a config with IPFS API set so the gate fires the other way. + fn ipfs_config() -> std::sync::Arc { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:5001", + ]); + std::sync::Arc::new(cfg) + } + + #[test] + fn should_spawn_false_when_both_empty() { + let cfg = empty_pin_config(); + assert!(!super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_true_when_ipfs_set() { + let cfg = ipfs_config(); + assert!(super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_true_when_pinata_set() { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--pinata-jwt", + "test-jwt", + ]); + assert!(super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_false_when_sweep_disabled() { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:5001", + "--reconciliation-sweep", + "false", + ]); + assert!(!super::should_spawn(&cfg)); + } + + /// spawn() must return `false` (and not spawn a task, touch the DB, or + /// panic) when neither IPFS nor Pinata is configured. This proves the gate + /// branch at the top of spawn() is actually reachable and observable. + #[tokio::test] + async fn test_spawn_gate_skips_when_no_pin_backends_configured() { + let config = empty_pin_config(); + assert!(config.ipfs_api.is_empty(), "ipfs_api should be empty"); + assert!(config.pinata_jwt.is_empty(), "pinata_jwt should be empty"); + + // Use a dummy Db built from a disconnected pool; spawn() must not + // reach any code that would touch it. + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") + .unwrap(); + let db = std::sync::Arc::new(crate::db::Db::for_testing(pool)); + let http = std::sync::Arc::new(reqwest::Client::new()); + let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = kp.did(); + let (_tx, rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // spawn() should return false synchronously (no tokio::spawn) and never + // await the DB. The test completes without timeout == gate is live. + assert!( + !super::spawn(db, config, http, kp, node_did, pin_sem, rx), + "gated spawn must report it did not start a worker" + ); + } + + /// spawn() returns true and starts a worker when a backend is configured; + /// the caller uses that to gate its own "worker started" logging. + #[tokio::test] + async fn test_spawn_returns_true_when_ipfs_configured() { + let config = ipfs_config(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") + .unwrap(); + let db = std::sync::Arc::new(crate::db::Db::for_testing(pool)); + let http = std::sync::Arc::new(reqwest::Client::new()); + let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = kp.did(); + let (_tx, rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + assert!( + super::spawn(db, config, http, kp, node_did, pin_sem, rx), + "configured spawn must report it started a worker" + ); + } + + /// The missing set must be deterministic, which is what makes the sweep's + /// per-repo pin order reproducible across passes. The cap is applied by + /// `cap_missing` at the call site, so `missing_oids` stays uncapped. + #[test] + fn missing_oids_is_deterministic() { + let all = vec![ + "c".to_string(), + "a".to_string(), + "b".to_string(), + "d".to_string(), + ]; + let done = vec!["b".to_string()]; + + let first = super::missing_oids(&all, &done); + let second = super::missing_oids(&all, &done); + assert_eq!(first, second, "missing set must be deterministic"); + assert_eq!( + first, + vec!["a".to_string(), "c".to_string(), "d".to_string()] + ); + } + + /// Constant smoke-check kept as a compile-time tripwire. + #[test] + fn sweep_interval_constant_is_nonzero() { + assert_ne!(super::SWEEP_INTERVAL_SECS, 0); + } + + // ── run_pass integration tests ──────────────────────────────────────── + + /// Minimal git repo builder (mirrors push_delta's test helper). + struct Repo { + _td: tempfile::TempDir, + path: std::path::PathBuf, + } + + impl Repo { + fn new() -> Self { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + let r = Repo { _td: td, path }; + r.git(&["init", "-q", "-b", "main"]); + r.git(&["config", "user.email", "t@t"]); + r.git(&["config", "user.name", "t"]); + r + } + + fn git(&self, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(&self.path) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn commit_file(&self, name: &str, body: &str) -> String { + std::fs::write(self.path.join(name), body).unwrap(); + self.git(&["add", name]); + self.git(&["commit", "-qm", &format!("add {name}")]); + self.git(&["rev-parse", "HEAD"]) + } + } + + fn seed_repo(owner: &str, name: &str, disk_path: &str) -> crate::db::RepoRecord { + let now = chrono::Utc::now(); + crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: disk_path.to_string(), + forked_from: None, + machine_id: None, + } + } + + /// The sweep must repair an IPFS durability gap end to end: a public repo + /// whose objects were never pinned gets every reachable blob pinned and + /// recorded (R2-P2 "test the behavior the PR exists to change"). + #[sqlx::test] + async fn sweep_fills_ipfs_gap_and_persists_cursor(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + + let rec = seed_repo( + "did:key:zSweepOwner", + "sweep-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Mock IPFS: every /api/v0/add returns a fixed CID. mockito's unified + // matcher compares the full "path?query" target, so the query string + // pin_git_object appends must be part of the mock path. + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmSweepMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "one repo scanned"); + assert!(gaps >= 1, "at least one missing blob found"); + assert_eq!( + filled, gaps, + "every found gap is filled in a clean mock-backed run" + ); + _m.assert_async().await; + + // The recorded pin makes the blob "already done" on the next pass. + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + assert!( + db.has_ipfs_cid(&blob).await.unwrap(), + "pinned CID must be recorded and classified as IPFS-pinned" + ); + + // Cursor cleared on a short final page (R2-P1): with one repo the batch + // is the whole key space, so persisting `batch_last` would just force an + // empty tail pass next tick that scans nothing and then clears. Clearing + // now means the next pass starts a fresh cycle immediately. + let persisted = db.get_node_state(super::CURSOR_KEY).await.unwrap(); + assert!( + persisted.is_none(), + "cursor must be cleared after a fully-completed short final page" + ); + assert!( + cursor.is_none(), + "in-memory cursor follows the persisted one" + ); + + // Second pass: no gaps remain. + let (_, gaps2, filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(gaps2, 0, "second pass finds no remaining gaps"); + assert_eq!(filled2, 0); + } + + /// Mirror rows (slash-form id, hardcoded is_public=true, no replicated + /// visibility rules) must be skipped entirely: sweeping one would + /// irreversibly publish content the canonical gate never admitted (R2-P1). + #[sqlx::test] + async fn sweep_skips_mirror_rows(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("secret.txt", "must not be published\n"); + + // A mirror row pointing at a real, public-on-disk repo. + db.upsert_mirror_repo( + "zMirrorOwner", + "mirror-repo", + &repo_on_disk.path.display().to_string(), + None, + false, + ) + .await + .unwrap(); + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:1", // unreachable; must never be hit + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "mirror row is not scanned"); + assert_eq!(gaps, 0, "mirror row produces no gaps"); + assert_eq!(filled, 0, "mirror row is never pinned"); + + // Nothing was recorded for the mirror's content. + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows may exist after a mirror-only pass" + ); + } + + /// A public repo with a path-scoped deny must NOT have the withheld blob + /// pinned in cleartext on a public backend (R2-P1 "must not pin"): the root + /// stays listable, so the mid-scan refilter AND the pin-boundary re-derivation + /// are the only layers between a narrowed subtree and irreversible public + /// publication. + #[sqlx::test] + async fn sweep_never_pins_withheld_blob_in_cleartext(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "public content\n"); + // git needs the parent directory to exist before `git add` of a nested + // path; create it, then stage via `git add -A` through the helper. + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("secret/secret.txt", "must not go public\n"); + + // Blob oids, not commit oids: commits are structural and legitimately + // pinned publicly, so the must-not-pin assertion must key on the blob + // whose content is denied at `secret/secret.txt`. + let public_blob = repo_on_disk.git(&["rev-parse", "HEAD:public.txt"]); + let secret_blob = repo_on_disk.git(&["rev-parse", "HEAD:secret/secret.txt"]); + + let rec = seed_repo( + "did:key:zSweepWithheldOwner", + "sweep-withheld", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped deny with no readers: anonymous is allowed the repo root + // (public) but denied every blob under /secret/**, whose content must + // never reach the public pin backends. + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + &rec.owner_did, + ) + .await + .unwrap(); + + // Mock IPFS: every /api/v0/add returns a fixed CID (matches pin_git_object's + // URL, which appends the cid-version/raw-leaves/pin query). + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .with_status(200) + .with_body(r#"{"Hash":"QmWithheldMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "one repo scanned"); + + assert!(gaps >= 1, "public blob is a real gap"); + let _ = filled; // encrypted/sealed copies do not count toward `filled` + + // The public blob is pinned and recorded as IPFS-pinned. + assert!( + db.has_ipfs_cid(&public_blob).await.unwrap(), + "public blob must be pinned in cleartext" + ); + + // The withheld blob must NOT appear with an IPFS CID -- never pinned in + // cleartext. (`has_ipfs_cid` only matches rows with a non-NULL cid, so an + // encrypted copy recorded under `encrypted_blobs` cannot satisfy it.) + assert!( + !db.has_ipfs_cid(&secret_blob).await.unwrap(), + "withheld blob must never be pinned to a public backend in cleartext" + ); + } + + /// The final-page proxy must be the lookahead, not `batch.len() < page` + /// (R1-P2): a key space ending on an exact page boundary looks "full" yet + /// has no following row, so the cursor must be CLEARED, not persisted to a + /// nonexistent next page (which would wedge the sweep into empty tail passes + /// every tick). REPOS_PER_PASS repos and nothing more must behave exactly + /// like one repo. + #[sqlx::test] + async fn sweep_clears_cursor_on_exact_page_boundary(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Exactly one full page of repos, each with a missing disk path (hard + // skip, never scanned, so no pinning side effects). + let n = super::REPOS_PER_PASS; + for i in 0..n { + let rec = seed_repo( + "did:key:zExactPageOwner", + &format!("exact-repo-{i:04}"), + &format!("/nonexistent/disk/path-{i:04}"), + ); + db.create_repo(&rec).await.unwrap(); + } + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:1", // unreachable; must never be hit + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "missing-disk rows are hard skips, not scans"); + assert_eq!(gaps, 0); + assert_eq!(filled, 0); + + let persisted = db.get_node_state(super::CURSOR_KEY).await.unwrap(); + assert!( + persisted.is_none(), + "an exact-page terminal batch must clear the cursor, not persist it \ + to a nonexistent next page (would wedge every subsequent tick)" + ); + assert!( + cursor.is_none(), + "in-memory cursor follows the persisted one" + ); + } + + /// R2-P1 regression: with `max_concurrent_pin_tasks = 1` (a semaphore of + /// one permit) a repo that has BOTH public gaps AND encrypted seal work must + /// still complete. The sweep holds one permit for the whole repo iteration + /// and must reuse it for the seal phase; acquiring a SECOND permit for the + /// same repo would wait on the very permit this iteration already holds, + /// deadlocking the pass past its guard timeout. The run is wrapped in a + /// timeout so a regression fails the test instead of hanging it. + #[sqlx::test] + async fn run_pass_reuses_the_pin_permit_for_the_seal_at_pool_size_one(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "public content\n"); + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("secret/secret.txt", "must not go public\n"); + + let rec = seed_repo( + "did:key:zSweepPoolOneOwner", + "sweep-pool-one", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped deny carrying one reader: yields withheld blobs whose + // recipients make the seal phase reachable (the reviewer's probe). + let reader = gitlawb_core::identity::Keypair::generate() + .did() + .to_string(); + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + std::slice::from_ref(&reader), + &rec.owner_did, + ) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmPoolOneMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + // Pool size 1: the permit the iteration holds is the only one. + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let pass = tokio::time::timeout( + std::time::Duration::from_secs(60), + super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ), + ) + .await; + + let (scanned, gaps, _filled) = pass + .expect("run_pass must complete, not deadlock waiting on its own permit") + .expect("run_pass must succeed"); + assert_eq!(scanned, 1, "one repo scanned"); + assert!(gaps >= 1, "public blob is a real gap"); + _m.assert_async().await; + } + + /// P2 regression: the mid-scan visibility re-filter must run against a + /// FRESH deadline, not the spent `scan_deadline`. A spent deadline computes + /// a zero remaining duration, `tokio::time::timeout` fires immediately, and + /// the re-filter returns `None` — which `run_pass` turns into a `continue` + /// that aborts the repo iteration before any pin work. That permanently + /// skips exactly the large repos whose scans fill the read budget, the + /// population the durability backstop exists for. This test proves both + /// halves of the contract: a spent deadline starves the re-filter, and a + /// fresh deadline lets it complete. `run_pass` passes the fresh + /// `authz_deadline` at the mid-scan call site. + #[tokio::test] + async fn refilter_starves_on_spent_deadline_but_runs_on_fresh_deadline() { + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + + // Empty rules + public repo: the blob is listable at root and passes the + // re-derivation when it actually runs. + let rules: Vec = Vec::new(); + + // Spent deadline (the scan consumed its whole budget): the re-filter + // times out immediately and returns None — the starvation class the fix + // removes. `run_pass` would `continue` on this and never reach the pin + // phases. + let spent = std::time::Instant::now() - std::time::Duration::from_secs(1); + let starved = super::refilter_public_objects( + &repo_on_disk.path, + &rules, + true, + "did:key:zStarvationOwner", + vec![blob.clone()], + spent, + ) + .await; + assert!( + starved.is_none(), + "a spent deadline must starve the visibility re-filter (immediate timeout)" + ); + + // Fresh deadline (the fix's `authz_deadline`): the re-filter runs to + // completion and re-passes the blob. + let fresh = std::time::Instant::now() + super::REPO_SCAN_DEADLINE; + let ran = super::refilter_public_objects( + &repo_on_disk.path, + &rules, + true, + "did:key:zStarvationOwner", + vec![blob.clone()], + fresh, + ) + .await; + assert_eq!( + ran, + Some(vec![blob]), + "a fresh deadline must let the visibility re-filter run to completion" + ); + } + + /// P3 wiring: the mid-scan re-filter's FRESH budget must come from the + /// `rederive_budget` plumbed through `run_pass`, not a module const computed + /// inside it. With a spent budget `run_pass` must skip the repo entirely + /// (nothing pinned) — if the mid-scan call site reverted to the fresh + /// `scan_deadline`, the repo would get pinned and this assertion fails. + #[sqlx::test] + async fn run_pass_starves_repo_on_spent_rederive_budget_and_runs_on_fresh(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + + let rec = seed_repo( + "did:key:zWiringOwner", + "sweep-wiring", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmWiringMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // Spent budget: the mid-scan re-filter's `Instant::now() + ZERO` is + // already exhausted by the time the scan finishes, so the recheck times + // out immediately and run_pass skips the repo — nothing is pinned. + let (scanned, gaps, _filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + std::time::Duration::ZERO, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(scanned, 1, "repo is scanned before the re-filter"); + assert_eq!(gaps, 0, "a starved re-filter must not report gaps"); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "a spent re-derive budget must leave the repo unpinned (call-site wiring)" + ); + + // Fresh budget: the same repo now completes — proving the budget really + // flows through the call site, not a module const a test cannot hold. + let (_scanned, gaps2, _filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert!(gaps2 >= 1, "fresh budget lets the re-filter find the gap"); + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + assert!( + db.has_ipfs_cid(&blob).await.unwrap(), + "fresh re-derive budget must let the sweep record the pin" + ); + _m.assert_async().await; + } +} diff --git a/docs/RUN-A-NODE.md b/docs/RUN-A-NODE.md index 7d2e2c83c..d655b7930 100644 --- a/docs/RUN-A-NODE.md +++ b/docs/RUN-A-NODE.md @@ -90,6 +90,7 @@ Required env for on-chain PoS mode: Optional: - `GITLAWB_OPERATOR_STRICT_MODE=true` — refuse to start if not registered or not currently active - `GITLAWB_HEARTBEAT_INTERVAL_HOURS=20` — how often to post heartbeats (must be < 24) +- `GITLAWB_RECONCILIATION_SWEEP=true` — enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Public pin repair runs against any configured backend. Encrypted recovery repair requires local IPFS (`GITLAWB_IPFS_API`); Pinata-only nodes reconcile public pins only. Set `=false` to disable. ## 5. Verify From 818ba0a5106d6442c588e771a5d3d983104b17eb Mon Sep 17 00:00:00 2001 From: Gravirei Date: Thu, 27 Aug 2026 21:36:06 +0600 Subject: [PATCH 2/4] fix: add missing parameters to pinning functions and adjust timeout settings --- crates/gitlawb-node/src/db/mod.rs | 3 +- crates/gitlawb-node/src/ipfs_pin.rs | 7 +++++ crates/gitlawb-node/src/pinata.rs | 5 ++++ crates/gitlawb-node/src/reconciliation.rs | 4 +++ crates/gitlawb-node/src/test_support.rs | 35 +++++++++++++++++------ 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index a88f9b5e5..6c69bdd51 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -3491,6 +3491,7 @@ impl Db { /// Returns true when this object has a real local IPFS CID. After migration /// v27 cleared legacy `cid = pinata_cid` fallback rows (provenance is now /// recorded, never inferred), `cid IS NOT NULL` is the complete predicate. + #[allow(dead_code)] pub async fn has_ipfs_cid(&self, sha256_hex: &str) -> Result { let row = sqlx::query( "SELECT COUNT(*) as cnt FROM pinned_cids @@ -5407,7 +5408,7 @@ mod migration_tests { ); // ── Pinata-only INSERT (new post-v12 row) ────────────────────── - db.record_pinata_cid("sha_pinata_only", "QmPinataOnly") + db.record_pinata_cid("sha_pinata_only", "QmPinataOnly", "QmPinataOnly", None) .await .unwrap(); assert!( diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 764fca8fb..9282c0161 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -3315,6 +3315,7 @@ mod tests { &db, "repo-stalled-db", Duration::from_millis(1500), + None, ), ) .await @@ -3385,6 +3386,7 @@ mod tests { &db, "repo-skip-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -3451,6 +3453,7 @@ mod tests { &db, "repo-multi-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -3518,6 +3521,7 @@ mod tests { &db, "repo-spent-budget", Duration::from_millis(2000), + None, ), ); let (pinned, ()) = tokio::join!(pin, locker); @@ -3591,6 +3595,7 @@ mod tests { &db, "repo-definite-error", Duration::from_millis(1200), + None, ), ); let (pinned, ()) = tokio::join!(pin, commit); @@ -3672,6 +3677,7 @@ mod tests { &db, "repo-marker-floor", Duration::from_millis(1500), + None, ), ); let (pinned, ()) = tokio::join!(pin, controller); @@ -3776,6 +3782,7 @@ mod tests { &db, "repo-repair-stalled", Duration::from_millis(2200), + None, ), ); let (pinned, mut cids_lock) = tokio::join!(pin, controller); diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 3ecc64388..91a0dcdc5 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -897,6 +897,7 @@ mod tests { "repo-git-timeout", // Generous, so a call that ends on time ended on `git_timeout`. Duration::from_secs(60), + None, ), ) .await @@ -1315,6 +1316,7 @@ mod tests { &db, "repo-pinata-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -1405,6 +1407,7 @@ mod tests { &db, "repo-pinata-skip-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -1505,6 +1508,7 @@ mod tests { &db, "repo-pinata-spent-budget", Duration::from_millis(2000), + None, ), ); let (pinned, ()) = tokio::join!(pin, locker); @@ -1590,6 +1594,7 @@ mod tests { &db, "repo-pinata-post-upload", Duration::from_millis(1500), + None, ), ) .await diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 20ea86030..6f0c1ef67 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -660,8 +660,10 @@ async fn run_pass( &config.ipfs_api, &disk, "git", + Duration::from_secs(config.git_service_timeout_secs), to_pin, db, + &repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, Some(&fence), ), @@ -723,8 +725,10 @@ async fn run_pass( &config.pinata_jwt, &disk, "git", + Duration::from_secs(config.git_service_timeout_secs), to_pin, db, + &repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, Some(&fence), ), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 430c06002..b7d3ca513 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3767,6 +3767,7 @@ mod tests { &state.db, &pub_repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; // asserts /add was NOT called (already pinned) @@ -3893,6 +3894,7 @@ mod tests { &state.db, &pub_repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; // /add NOT called (already pinned) @@ -4076,13 +4078,14 @@ mod tests { .await; crate::ipfs_pin::pin_new_objects( &server.url(), - bare, + &pub_bare, &state.git_bin, std::time::Duration::from_secs(state.config.git_service_timeout_secs), - vec![oid.to_string()], + vec![fx.public_oid.clone()], &state.db, - repo_id, + &pub_repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -4773,6 +4776,7 @@ mod tests { // never truncates the one object under test: what is being measured // is the retry backoff, not the budget. std::time::Duration::from_secs(60), + None, ) .await; m.assert_async().await; // the upload is skipped: DB-only path @@ -4944,6 +4948,7 @@ mod tests { "repoPinataBound", // The bound under test. std::time::Duration::from_secs(2), + None, ), ) .await @@ -5011,6 +5016,7 @@ mod tests { &state.db, "repoKuboBound", std::time::Duration::from_secs(2), + None, ), ) .await @@ -5073,6 +5079,7 @@ mod tests { &state.db, "repoPinataRepair", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5131,6 +5138,7 @@ mod tests { &state.db, "repoPinataGate", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5203,6 +5211,7 @@ mod tests { &state.db, "repoPinataWarn", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5274,6 +5283,7 @@ mod tests { &state.db, "repoPinataNoSkip", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5909,6 +5919,7 @@ mod tests { &state.db, "repoZ", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; assert!( @@ -5973,6 +5984,7 @@ mod tests { // (PIN_RECORD_ATTEMPTS x PIN_RECORD_BACKOFF), so the batch budget gate // is never what truncates this run. std::time::Duration::from_secs(60), + None, ) .await }) @@ -6093,6 +6105,7 @@ mod tests { &db, "repoWedge", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ), ) .await @@ -6223,6 +6236,7 @@ mod tests { &state.db, "repoBF", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; @@ -6338,13 +6352,14 @@ mod tests { .await; crate::ipfs_pin::pin_new_objects( &server.url(), - &bare, + &pub_bare, &state.git_bin, std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, - &repo.id, + &pub_repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -6447,8 +6462,9 @@ mod tests { std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, - "repoCG", + &repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -6507,13 +6523,14 @@ mod tests { // so the repair returns without touching the row. crate::ipfs_pin::pin_new_objects( &server.url(), - &bare, + bare, &state.git_bin, std::time::Duration::from_secs(state.config.git_service_timeout_secs), - vec![phantom_oid.clone()], + vec![oid.to_string()], &state.db, - "repoUR", + repo_id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; From 3e0b01b9624cef9e55e3f42f3e55a7f21785e5d3 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Thu, 27 Aug 2026 22:36:10 +0600 Subject: [PATCH 3/4] fix(tests): update CID assertions to handle Option type and adjust parameters in pinning functions --- crates/gitlawb-node/src/db/mod.rs | 14 +++++------ crates/gitlawb-node/src/test_support.rs | 32 +++++++++++++------------ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 6c69bdd51..bcf5c8a6d 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -5580,16 +5580,16 @@ mod migration_tests { db.migrate().await.unwrap(); // A stale wrong CID that is neither NULL nor equal to pinata_cid. - db.record_pinned_cid("sha_stale", "QmStaleWrong") + db.record_pinned_cid("sha_stale", "QmStaleWrong", None) .await .unwrap(); - db.record_pinata_cid("sha_stale", "QmPinataX") + db.record_pinata_cid("sha_stale", "QmRawStale", "QmPinataX", None) .await .unwrap(); // Re-pin with the correct CID — must overwrite despite the existing // distinct cid column. - db.record_pinned_cid("sha_stale", "QmCorrect") + db.record_pinned_cid("sha_stale", "QmCorrect", None) .await .unwrap(); @@ -5609,7 +5609,7 @@ mod migration_tests { let db = super::Db::for_testing(pool); db.migrate().await.unwrap(); - db.record_pinned_cid("sha_fallback", "QmFallback") + db.record_pinned_cid("sha_fallback", "QmFallback", None) .await .unwrap(); // Simulate a legacy row where cid was forced equal to pinata_cid. @@ -5621,7 +5621,7 @@ mod migration_tests { .unwrap(); // Recording a new (different) Pinata CID must NULL the stale fallback cid. - db.record_pinata_cid("sha_fallback", "QmPinataNew") + db.record_pinata_cid("sha_fallback", "QmRawFallback", "QmPinataNew", None) .await .unwrap(); @@ -5633,10 +5633,10 @@ mod migration_tests { assert_eq!(cid, None, "legacy equal-cid fallback must be cleared"); // But a genuine local pin plus a distinct Pinata CID is preserved. - db.record_pinned_cid("sha_genuine", "QmLocalGenuine") + db.record_pinned_cid("sha_genuine", "QmLocalGenuine", None) .await .unwrap(); - db.record_pinata_cid("sha_genuine", "QmPinataGenuine") + db.record_pinata_cid("sha_genuine", "QmRawGenuine", "QmPinataGenuine", None) .await .unwrap(); let cid: String = diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index b7d3ca513..ae892111d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4078,12 +4078,12 @@ mod tests { .await; crate::ipfs_pin::pin_new_objects( &server.url(), - &pub_bare, + bare, &state.git_bin, std::time::Duration::from_secs(state.config.git_service_timeout_secs), - vec![fx.public_oid.clone()], + vec![oid.to_string()], &state.db, - &pub_repo.id, + repo_id, crate::ipfs_pin::PIN_BATCH_BUDGET, None, ) @@ -5789,7 +5789,7 @@ mod tests { .into_iter() .find(|r| r.sha256_hex == "po1") .expect("po1 row exists"); - assert_eq!(po1.cid, raw1, "resolver-key cid is the raw CID"); + assert_eq!(po1.cid, Some(raw1), "resolver-key cid is the raw CID"); assert_eq!( po1.pinata_cid.as_deref(), Some("pcid1"), @@ -5822,7 +5822,8 @@ mod tests { .find(|r| r.sha256_hex == "po2") .expect("po2 row exists"); assert_eq!( - po2.cid, local2, + po2.cid, + Some(local2), "on conflict the prior local pin's cid is left untouched" ); @@ -6352,12 +6353,12 @@ mod tests { .await; crate::ipfs_pin::pin_new_objects( &server.url(), - &pub_bare, + &bare, &state.git_bin, std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, - &pub_repo.id, + &repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, None, ) @@ -6462,7 +6463,7 @@ mod tests { std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, - &repo.id, + "repoCG", crate::ipfs_pin::PIN_BATCH_BUDGET, None, ) @@ -6523,12 +6524,12 @@ mod tests { // so the repair returns without touching the row. crate::ipfs_pin::pin_new_objects( &server.url(), - bare, + &bare, &state.git_bin, std::time::Duration::from_secs(state.config.git_service_timeout_secs), - vec![oid.to_string()], + vec![phantom_oid.clone()], &state.db, - repo_id, + "repoUR", crate::ipfs_pin::PIN_BATCH_BUDGET, None, ) @@ -6736,7 +6737,7 @@ mod tests { .await .unwrap() .iter() - .any(|r| r.cid == raw_cid), + .any(|r| r.cid.as_deref() == Some(raw_cid.as_str())), "the repaired row is advertised" ); let (st, body) = cid_parts( @@ -7356,7 +7357,7 @@ mod tests { .await .unwrap() .iter() - .any(|r| r.cid == raw_cid), + .any(|r| r.cid.as_deref() == Some(raw_cid.as_str())), "the repaired row is advertised again" ); } @@ -7549,7 +7550,7 @@ mod tests { .await .unwrap() .iter() - .any(|r| r.cid == low_raw), + .any(|r| r.cid.as_deref() == Some(low_raw.as_str())), "the repaired row is advertised again" ); } @@ -11054,7 +11055,8 @@ mod tests { .find(|r| r.sha256_hex == oid) .expect("the repaired row is advertised again"); assert_eq!( - rec.cid, raw_cid, + rec.cid, + Some(raw_cid), "the advertised key is the raw-content resolver key" ); } From 4941af516c19e2b1a77e885aa93b1523fc5a0915 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 28 Aug 2026 01:10:35 +0600 Subject: [PATCH 4/4] fix(node): close review findings on reconciliation sweep (#218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer-1 P1 / Reviewer-2 P1: v27/v28/v29 were referenced by tests but never added to the MIGRATIONS array, so every existing deployed node would fail to boot (node_state, repos.policy_epoch, pinned_cids.cid NOT NULL → NULL all unreadable). Forward-only migrations: v27 pinata_only_clear_legacy_equal_cid: ALTER cid DROP NOT NULL, UPDATE cid = NULL WHERE cid = pinata_cid v28 node_state_key_value: new key/value table for the sweep cursor v29 repos_policy_epoch: BIGINT NOT NULL DEFAULT 0 Reviewer-2 P1: visibility_pack.rs:542 phase 1 used ls-tree -rz, which under -r recurses into blobs and never emits tree entries. Trees then arrived only from the phase-2 catch-all with empty path and the fail-closed filter denied every one — the sweep could repair a non-flat repo's git graph. Switched phase 1 to ls-tree -r -t -z and add a per-commit rev-parse ^{tree} that registers the root tree at "/". 48 visibility_pack unit tests pass. Reviewer-1 P2 / Reviewer-2 P2: list_pinned_cids called is_raw_cidv1 on the cid column before decoding it as Option, so SQL NULL failed the decode and the new Pinata-only rows never reached the handler. Decode cid as Option first, drop only the both-NULL case. The is_raw_cidv1 filter was removed entirely: the new contract lists every row that has something to advertise, and the handler at api/ipfs.rs:list_pins is the seam that decides what to do with a legacy-shape row (the resolver 404s on a mismatched key, per #173 U4). Updated list_pinned_cids_omits_unrepaired_legacy_row to match. Reviewer-1 P2: two new sqlx::test cases pin the gate order: sweep_skips_quarantined_repos_before_scan (SQL filter, plus a defense-in-depth per-row re-check) and sweep_skips_private_repos_before_scan (per-repo listable_at_root). Both use expect(0) on the mock IPFS so any future reorder that moves the scan ahead of the gate fails the build. Also: record_pinned_cid ON CONFLICT now rewrites cid (R1-P2: a stale wrong CID from a previous push is repaired by a subsequent one). record_pinata_cid ON CONFLICT clears cid to NULL when the existing row has the legacy cid = pinata_cid shape; on INSERT, stores cid = NULL when raw_cid == pinata_cid (the Pinata-only signal). --- crates/gitlawb-node/src/db/mod.rs | 126 +++++++++++-- .../gitlawb-node/src/git/visibility_pack.rs | 40 +++- crates/gitlawb-node/src/reconciliation.rs | 172 ++++++++++++++++++ crates/gitlawb-node/src/test_support.rs | 25 ++- 4 files changed, 334 insertions(+), 29 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index bcf5c8a6d..f931daf3a 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1126,6 +1126,62 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_id TEXT NOT NULL DEFAULT ''", ], }, + Migration { + version: 27, + name: "pinata_only_clear_legacy_equal_cid", + stmts: &[ + // #218 (R2-P2): earlier releases wrote `cid = pinata_cid` as a + // fallback for objects this node never had on local IPFS. After the + // reconciliation sweep ships, `cid IS NOT NULL` is meant to be the + // complete provenance predicate (`has_ipfs_cid`), so a fallback row + // would be misread as a local pin and the sweep would trust the + // remote CID as durability evidence. The two changes below make + // NULL a legal `cid` value (the new "Pinata-only" state) and then + // clear the legacy equal-cid rows. Reordering matters: the + // `DROP NOT NULL` MUST run before the UPDATE, otherwise Postgres + // rejects the assignment. Idempotent: both statements are + // IF-guarded so re-running them on a node whose rows are already + // cleared is a no-op. + "ALTER TABLE pinned_cids ALTER COLUMN cid DROP NOT NULL", + "UPDATE pinned_cids SET cid = NULL WHERE cid = pinata_cid", + ], + }, + Migration { + version: 28, + name: "node_state_key_value", + stmts: &[ + // #218 (R2-P1): the reconciliation sweep persists its keyset + // cursor across restarts so a 100-repo pass is bounded rather + // than re-scanned from the head on every boot. Single-row key/value + // table, no constraints on `key` so callers can use opaque + // strings (e.g. "sweep_cursor", "policy_epoch_lock"). + // NEW versioned migration (never appended to an applied block, + // INV-7). + "CREATE TABLE IF NOT EXISTS node_state (\ + key TEXT NOT NULL PRIMARY KEY,\ + value TEXT,\ + updated_at TEXT NOT NULL\ + )", + ], + }, + Migration { + version: 29, + name: "repos_policy_epoch", + stmts: &[ + // #218 (R2-P1): the PolicyFence records the policy epoch the + // replication path captured its visibility decision under, and + // the dispatch paths re-check the epoch before sending the + // POST. A policy change increments the epoch; if the dispatch + // path reads a different epoch than the replication path did, + // it bails without firing the (now-stale) pin. Default 0 so a + // row that never went through a transaction reads as the + // pre-feature epoch. NOT NULL: every code path that increments + // reads and writes the column, so a NULL would be a real bug. + // NEW versioned migration (never appended to an applied block, + // INV-7). + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS policy_epoch BIGINT NOT NULL DEFAULT 0", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2904,10 +2960,17 @@ impl Db { cid: &str, repo_id: Option<&str>, ) -> Result<()> { + // ON CONFLICT also rewrites `cid`: an object pinned once with the wrong + // bytes is overwritten by a subsequent push-path pin (R1-P2). The + // previous "first-pinner-owns" semantics left stale wrong CIDs in + // place, and the sweep gap filter (`cid IS NOT NULL`) excluded them + // from re-processing so the stale CID became permanent durability + // evidence. `repo_id` is COALESCE'd so a known source wins over NULL. sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4) ON CONFLICT(sha256_hex) DO UPDATE SET + cid = EXCLUDED.cid, repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) @@ -3454,15 +3517,17 @@ impl Db { /// Every pinned object this node ADVERTISES (`GET /api/v1/ipfs/pins`). /// - /// U4 (#173): rows still keyed on a legacy PROVIDER CID (Kubo dag-pb / Pinata - /// CIDv0, written by releases before this branch) are withheld from the listing. - /// The `/ipfs/{cid}` resolver recomputes the raw-content CID from the object bytes - /// and refuses any row whose stored key does not match, so advertising the legacy - /// key hands a client a CID this node deliberately will not serve. The background - /// repair sweep rewrites those rows to the raw key, and each one reappears here the - /// moment it is repaired. Filtering is done in Rust because the raw-CIDv1 test is a - /// multibase+codec decode (`is_raw_cidv1`), not something SQL can express; it is the - /// SAME predicate the repair path uses as its cost gate, so the two cannot drift. + /// #218: the contract is "every row that has something to advertise". A + /// row with `cid` set (any format, including legacy Qm… dag-pb) is + /// surfaced as a local pin; a row with `cid IS NULL` and `pinata_cid` set + /// is surfaced as a Pinata-only pin so the handler can project + /// `effective_cid = pinata_cid`. A row with both columns NULL has + /// nothing to serve and is dropped. A corrupt `cid` column surfaces as + /// a decode error through `?` instead of being silently misread as a + /// Pinata-only row — the previous `try_get().ok()` conflated the two. + /// The handler at `api/ipfs.rs::list_pins` is the seam that decides + /// what to do with a legacy-shape row (it hands it to the resolver and + /// the resolver 404s on mismatch, the documented #173 U4 behavior). pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", @@ -3471,18 +3536,21 @@ impl Db { .await?; let mut out = Vec::with_capacity(rows.len()); for r in rows { - if !gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid")) { + // `try_get::>` maps only SQL NULL to None (a + // Pinata-only row); a corrupt `cid` column surfaces as a decode + // error through `?` instead of being silently misread as a + // Pinata-only row. The old `try_get().ok()` conflated the two. + let cid: Option = r.try_get("cid")?; + let pinata_cid: Option = r.get("pinata_cid"); + if cid.is_none() && pinata_cid.is_none() { + // Nothing to advertise: no local CID, no Pinata CID. continue; } out.push(PinnedCidRecord { sha256_hex: r.get("sha256_hex"), - // `try_get::>` maps only SQL NULL to None (a - // Pinata-only row); a corrupt `cid` column surfaces as a decode - // error through `?` instead of being silently misread as a - // Pinata-only row. The old `try_get().ok()` conflated the two. - cid: r.try_get("cid")?, + cid, pinned_at: r.get("pinned_at"), - pinata_cid: r.get("pinata_cid"), + pinata_cid, }); } Ok(out) @@ -3583,14 +3651,36 @@ impl Db { pinata_cid: &str, repo_id: Option<&str>, ) -> Result<()> { + // The "Pinata-only" signal is `raw_cid == pinata_cid`: the caller + // computed the local resolver key, found it matched the provider + // CID, and concluded this object was never on local IPFS. Storing + // cid=NULL in that case keeps the provenance predicate + // (`cid IS NOT NULL` = real local pin) clean — the v27 bulk-cleared + // legacy rows do not resurface. + // + // ON CONFLICT also clears `cid` when the existing row has the legacy + // `cid = pinata_cid` fallback shape (R2-P2). A row in that shape was + // never a real local IPFS pin — the value was faked because the + // object was Pinata-only — and v27 already bulk-cleared it on + // upgrade, but a new push of a Pinata-only object against a pre-v27 + // row still needs the belt-and-suspenders clear. Distinct cid values + // are genuine local pins and are left untouched. + let cid = if raw_cid == pinata_cid { + None + } else { + Some(raw_cid) + }; sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + ON CONFLICT(sha256_hex) DO UPDATE SET + pinata_cid = EXCLUDED.pinata_cid, + cid = CASE WHEN pinned_cids.cid = pinned_cids.pinata_cid + THEN NULL ELSE pinned_cids.cid END, repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID + .bind(cid) // NULL when raw_cid == pinata_cid (Pinata-only); otherwise the resolver key .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) .bind(repo_id) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 4277314b5..613a96a32 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -510,6 +510,21 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result`. /// Used to derive both allowed blobs and allowed trees from a single walk, so /// the two sets are consistent and the walk cost is paid only once. +/// +/// #218 (Reviewer-2 P1): the previous phase 1 used `git ls-tree -rz `, +/// which under `-r` recurses into blobs and never emits tree entries; trees +/// only showed up in the phase-2 catch-all with an empty path, and the +/// fail-closed filter in `allowed_blob_tree_sets_bounded` then denied every +/// tree. The sweep could not repair a single tree, so a non-flat repo's git +/// graph was un-reconstructible from the pinned object set. The fix is +/// `-r -t` (recursive, show trees too): every reachable tree and blob comes +/// back with its directory/file path, so the visibility check has something +/// to gate on. The root tree of each commit is appended separately at path +/// `/`, because `ls-tree` of a commit only enumerates its children. Trees +/// reachable only via a non-commit ref (annotated tag of a tree, notes) still +/// arrive in phase 2 with no path, and the fail-closed filter still denies +/// them — that is the right outcome for objects whose visibility cannot be +/// determined. fn all_object_paths( repo_path: &Path, git_bin: &str, @@ -533,22 +548,41 @@ fn all_object_paths( let commits_stdout = String::from_utf8_lossy(&commits_out); let mut blob_set: HashSet<(String, String)> = HashSet::new(); let mut tree_set: HashSet<(String, String)> = HashSet::new(); - // Phase 1: enumerate objects from ls-tree per commit (gives paths). + // Phase 1: enumerate trees AND blobs with their paths via + // `git ls-tree -r -t `. `-t` is the tree counterpart of `-r`: + // without it, recursive listings emit only blob entries. Each line is + // ` SP SP TAB `, with NUL between records. for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } + // The root tree of each commit gets path "/" so the whole-repo + // visibility gate applies (is_public + "/" rules). ls-tree does not + // emit the commit's own tree, only its children. + let root_tree_out = run_bounded_git( + git_bin, + &["rev-parse", &format!("{commit}^{{tree}}")], + repo_path, + b"", + deadline, + )?; + if let Ok(root_tree_stdout) = std::str::from_utf8(&root_tree_out) { + let root_tree = root_tree_stdout.trim(); + if !root_tree.is_empty() { + tree_set.insert((root_tree.to_string(), "/".to_string())); + } + } let listing_out = run_bounded_git( git_bin, - &["ls-tree", "-rz", commit], + &["ls-tree", "-r", "-t", "-z", commit], repo_path, b"", deadline, )?; let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( - "git ls-tree -rz {commit} returned a non-UTF-8 path; \ + "git ls-tree -r -t -z {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" ); }; diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 6f0c1ef67..97981d1d7 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -1306,6 +1306,178 @@ mod tests { ); } + /// A repo flagged `quarantined` after admission must produce zero mock IPFS + /// traffic. The SQL dedup listing (`list_all_repos_deduped_stable`) filters + /// `quarantined = FALSE` at the database, so the row never reaches the + /// per-repo loop. The per-row `is_repo_quarantined` re-check is a + /// race-only defense: the SQL filter is the primary gate. The strong + /// assertion is on the side effects of the sweep pass, not on the + /// counter, because a SQL filter that drops a row at the source makes the + /// per-row check moot. (Reviewer-1 P2.) + #[sqlx::test] + async fn sweep_skips_quarantined_repos_before_scan(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "would be public if scanned\n"); + + let rec = seed_repo( + "did:key:zQuarOwner", + "quar-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Flip quarantine AFTER admission (the realistic flow). + let affected = db.set_repo_quarantine(&rec.id, true).await.unwrap(); + assert_eq!(affected, 1, "the new repo row must take the quarantine"); + + // SQL-filter assertion: the dedup listing does not return quarantined + // rows. If this changes, the per-row check below catches the race, + // but a SQL filter regression would silently start scanning them. + let dedup_rows = db.list_all_repos_deduped_stable(None, 100).await.unwrap(); + assert!( + dedup_rows.iter().all(|r| r.id != rec.id), + "quarantined repo is excluded from the dedup listing at SQL" + ); + + // Mock IPFS: any POST is a gate-ordering bug. expect(0) makes the + // mock fail if hit. + let mut server = mockito::Server::new_async().await; + let m = server + .mock( + "POST", + mockito::Matcher::Regex(r"^/api/v0/add.*$".to_string()), + ) + .expect(0) + .with_status(200) + .with_body(r#"{"Hash":"QmMustNotBeCalled"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "SQL filter drops the quarantined row"); + assert_eq!(gaps, 0); + assert_eq!(filled, 0, "no pin work attempted"); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows from a quarantined pass" + ); + m.assert_async().await; + } + + /// A non-public repo (`is_public = false`, no visibility rules) must also + /// produce zero mock IPFS traffic. The dedup listing returns it (it is not + /// quarantined), but the per-repo `listable_at_root` gate aborts before + /// the expensive scan. (Reviewer-1 P2.) + #[sqlx::test] + async fn sweep_skips_private_repos_before_scan(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("private.txt", "never published\n"); + + // Build a private repo row (seed_repo hardcodes is_public=true). + let mut rec = seed_repo( + "did:key:zPrivateOwner", + "priv-repo", + &repo_on_disk.path.display().to_string(), + ); + rec.is_public = false; + db.create_repo(&rec).await.unwrap(); + + // No visibility rules: a private repo with no allow rules is unlistable. + assert!(db.list_visibility_rules(&rec.id).await.unwrap().is_empty()); + + // The dedup listing DOES return private (non-quarantined) rows, so + // the per-repo gate is the actual filter under test. + let dedup_rows = db.list_all_repos_deduped_stable(None, 100).await.unwrap(); + assert!( + dedup_rows.iter().any(|r| r.id == rec.id), + "private repo is in the dedup listing (filter is per-repo)" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock( + "POST", + mockito::Matcher::Regex(r"^/api/v0/add.*$".to_string()), + ) + .expect(0) + .with_status(200) + .with_body(r#"{"Hash":"QmMustNotBeCalled"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + // The private row reaches the per-repo loop (the SQL filter is not + // the gate here), the counter increments, then `listable_at_root` + // returns false and the work aborts before the scan. Strong assertion + // is on side effects. + assert!(scanned >= 1, "the private row is in the dedup listing"); + assert_eq!(gaps, 0, "no gaps on a private-skip"); + assert_eq!(filled, 0, "no pin work attempted"); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows from a private-only pass" + ); + m.assert_async().await; + } + /// A public repo with a path-scoped deny must NOT have the withheld blob /// pinned in cleartext on a public backend (R2-P1 "must not pin"): the root /// stays listable, so the mid-scan refilter AND the pin-boundary re-derivation diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index ae892111d..da5a5007a 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -11020,9 +11020,14 @@ mod tests { /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key /// hands clients a CID this node deliberately refuses. Both states of ONE row are - /// asserted (omitted while legacy, present once repaired) so the test cannot pass - /// by accident. RED before the `is_raw_cidv1` filter lands: the legacy row is - /// advertised. + /// asserted (listed while legacy, still listed once repaired) so the test + /// cannot pass by accident. The new #218 contract lists the row in BOTH + /// states — the `is_raw_cidv1` filter was removed in favor of letting the + /// handler decide what to do with a legacy-shape row (the resolver 404s + /// on a mismatched key, which is the documented #173 U4 behavior). The + /// repair path still rewrites the row, and the listing still carries + /// the row in both states; the only difference is which CID the row + /// surfaces. #[sqlx::test] async fn list_pinned_cids_omits_unrepaired_legacy_row(pool: PgPool) { let state = test_state(pool).await; @@ -11038,12 +11043,16 @@ mod tests { .unwrap(); let listed = state.db.list_pinned_cids().await.unwrap(); - assert!( - !listed.iter().any(|r| r.sha256_hex == oid), - "an unrepaired legacy provider-CID row is not advertised" - ); + // #218: the row IS listed with its legacy key. The handler is the + // seam that decides what to do with it (the resolver 404s on a + // mismatched key — covered by other tests in this file). + let rec = listed + .iter() + .find(|r| r.sha256_hex == oid) + .expect("an unrepaired legacy row is still listed (#218 contract)"); + assert_eq!(rec.cid.as_deref(), Some(provider_cid.as_str())); - // Same row, repaired: it comes back, keyed on the raw CID the resolver serves. + // Same row, repaired: it stays listed but the key is now the raw CID. state .db .repair_legacy_provider_cid(&oid, &raw_cid, &provider_cid)