Anchor the witnessed journal in Bitcoin from main: OTS proofs, tool, and daily job (supersedes #182, #183)

.github/workflows/ci.yml

@@ -37,6 +37,14 @@ jobs:
         run: >
           uv run pytest -q
 
+      # Keep this security-sensitive operational surface explicit even though
+      # the broad Chronicle lint and test commands above also cover it.
+      - name: Lint OpenTimestamps anchoring
+        run: uv run ruff check scripts/ots_anchor.py tests/test_ots_anchor.py
+
+      - name: Test OpenTimestamps anchoring
+        run: uv run pytest -q tests/test_ots_anchor.py
+
       - name: Build source input database
         run: |
           uv run chronicle --db /tmp/chronicle-targets-ci.db init

.github/workflows/ots-anchor.yml

@@ -0,0 +1,177 @@
+name: OTS anchor
+
+# The trusted tool and mutable proofs live on main. The journal checkout is
+# credential-free input only; no code from it is executed and it is never pushed.
+
+on:
+  schedule:
+    - cron: "23 6 * * *"
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: ots-anchor-main
+  cancel-in-progress: false
+
+jobs:
+  anchor:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      # API evidence on 2026-09-02 showed no effective rules, required checks,
+      # reviews, or push restrictions blocking a GITHUB_TOKEN update to main.
+      contents: write
+    env:
+      MAIN_BRANCH: main
+      JOURNAL_BRANCH: codex/thesis-ledger-facts
+      OTS_BIN: uvx --from opentimestamps-client==0.7.2 ots
+    steps:
+      - name: Log main publication rule evidence
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -uo pipefail
+
+          log_api() {
+            endpoint="$1"
+            printf 'gh api %s\n' "$endpoint"
+            if ! gh api "$endpoint"; then
+              printf '::warning::Could not read %s; the bounded push remains authoritative.\n' \
+                "$endpoint" >&2
+            fi
+          }
+
+          log_api "repos/$GITHUB_REPOSITORY/rules/branches/$MAIN_BRANCH"
+          log_api "repos/$GITHUB_REPOSITORY/rulesets"
+          log_api "repos/$GITHUB_REPOSITORY/branches/$MAIN_BRANCH/protection"
+
+      - name: Check out trusted main
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: main
+          fetch-depth: 0
+          persist-credentials: true
+
+      - name: Check out journal manifests without credentials
+        uses: actions/checkout@v4
+        with:
+          ref: codex/thesis-ledger-facts
+          path: journal
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Anchor, verify, and publish proof-only changes
+        working-directory: main
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          manifest_dir="$GITHUB_WORKSPACE/journal/releases/manifests"
+
+          anchor_and_verify() {
+            python3 scripts/ots_anchor.py run \
+              --manifests "$manifest_dir" \
+              --ots-bin "$OTS_BIN"
+            python3 scripts/ots_anchor.py verify \
+              --manifests "$manifest_dir" \
+              --ots-bin "$OTS_BIN"
+            python3 scripts/ots_anchor.py guard
+          }
+
+          refresh_journal() {
+            git -C "$GITHUB_WORKSPACE/journal" fetch \
+              --no-tags --depth=1 origin "$JOURNAL_BRANCH"
+            git -C "$GITHUB_WORKSPACE/journal" checkout \
+              --detach FETCH_HEAD
+          }
+
+          commit_proofs() {
+            git add -- ots/
+            if git diff --cached --quiet; then
+              return
+            fi
+            if git diff --quiet "origin/$MAIN_BRANCH"...HEAD; then
+              git commit \
+                -m "Update OpenTimestamps anchors for release manifests" \
+                -m "Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>"
+            else
+              git commit --amend --no-edit
+            fi
+          }
+
+          assert_commit_scope() {
+            outside=0
+            while IFS= read -r changed_path; do
+              if [[ -z "$changed_path" ]]; then
+                continue
+              fi
+              case "$changed_path" in
+                ots/*) ;;
+                *)
+                  printf 'refusing committed path outside ots/: %s\n' \
+                    "$changed_path" >&2
+                  outside=1
+                  ;;
+              esac
+            done < <(git diff --name-only "origin/$MAIN_BRANCH"...HEAD)
+            if [[ "$outside" -ne 0 ]]; then
+              return 1
+            fi
+            if [[ -n "$(git status --porcelain)" ]]; then
+              git status --short >&2
+              printf 'refusing to push a dirty worktree\n' >&2
+              return 1
+            fi
+          }
+
+          git config user.name "github-actions[bot]"
+          git config user.email \
+            "41898282+github-actions[bot]@users.noreply.github.com"
+
+          anchor_and_verify
+          commit_proofs
+          if git diff --quiet "origin/$MAIN_BRANCH"...HEAD; then
+            printf 'no proof changes to publish\n'
+            exit 0
+          fi
+          assert_commit_scope
+
+          for attempt in 1 2 3; do
+            if push_output="$(git push origin HEAD:main 2>&1)"; then
+              printf '%s\n' "$push_output"
+              exit 0
+            fi
+            printf '%s\n' "$push_output" >&2
+            if ! git fetch --no-tags origin main; then
+              printf 'push failed and main could not be fetched for retry analysis\n' >&2
+              exit 1
+            fi
+            if git merge-base --is-ancestor HEAD origin/main; then
+              printf 'local proof commit is already contained in main\n'
+              exit 0
+            fi
+            if git merge-base --is-ancestor origin/main HEAD; then
+              printf 'push failed for a reason other than non-fast-forward\n' >&2
+              exit 1
+            fi
+            if [[ "$attempt" -eq 3 ]]; then
+              printf 'push remained non-fast-forward after 3 attempts\n' >&2
+              exit 1
+            fi
+
+            git rebase origin/main
+            refresh_journal
+            anchor_and_verify
+            commit_proofs
+            if git diff --quiet origin/main...HEAD; then
+              printf 'proof changes already landed during retry\n'
+              exit 0
+            fi
+            assert_commit_scope
+          done

PROGRESS.md

@@ -1,51 +1,68 @@
-# Lane C5 progress
+# PR #229 completion
 
 ## State
 
-- Branch: `be-2025-vintages` from `origin/main` at `5c15bfd`.
-- Worktree inputs are staged under `.lane-raw/` and must remain uncommitted.
-- Lane C5 is complete, validated, independently reviewed, and ready for handoff.
-- The requested staged C2 report is absent, but root `LANE_C2_REPORT.md` is byte-identical
-  to the sibling lane's staged copy (SHA-256 `4590e0dc...50f06e7`) and is the pattern used.
+The pending-proof classification fix, direct-push publication workflow, final
+repository verification, and report are complete. GitHub delivery is blocked:
+shell DNS prevents fetch/push/`gh pr edit`, and the authenticated connector's PR
+body mutation was cancelled. PR #229 remains open and unmerged.
 
 ## Done
 
-- Read the repository Chronicle boundary rules in `AGENTS.md`.
-- Read `.lane-raw/SOURCES.md` and confirmed all five named publisher artifacts are present.
-- Confirmed the worktree is otherwise clean apart from `.lane-raw/` and the shared `.venv` link.
-- Verified all five staged artifact SHA-256 pins exactly.
-- Mapped FPB workbook cells: 990 facts across T01/T06/T07/T11/T17/T24, with
-  2022–2025 observations and 2026–2031 `source_projection` facts.
-- Confirmed PDF boundary evidence: printed page 19 calls 2026 the first projection year;
-  annex table units appear on printed pages 45, 48, 49, 53, 58, and 65.
-- Chosen Eurostat layout: two vintage-specific source-package aliases share new manifest
-  entries, preserving the prior package YAMLs, raw bytes, and fact outputs unchanged.
-- Reproduced the Statbel curator logic: 18 NUTS1 × sex × age-band cells totaling 11,825,551.
-- Added the hash-pinned FPB workbook and publication PDF plus the
-  `fpb-economic-outlook-2026-2031-june-2026` package alias.
-- Built 990 line-specific publisher facts (99 per year): 396 observations for
-  2022–2025 and 594 `source_projection` facts for 2026–2031.
-- Passed FPB `validate-package` and `build-suite`: 990 facts, full cell lineage,
-  zero acceptance errors, and pinned 2025 cells 320578 / 77771 / 5602 million euro.
-- Re-ran the Statbel 2026 curator logic on the 2025 ZIP and added the hash-pinned
-  raw capture plus its deterministic 18-row curated CSV.
-- Passed Statbel 2025 `validate-package` and `build-suite`: 18 facts totaling
-  11,825,551, 66 constraints, full lineage, and zero acceptance errors.
-- Added the Eurostat `gov_10a_taxag` 2025 and `spr_exp_func` 2024 manifest
-  entries plus vintage-specific package aliases, without modifying either
-  prior artifact or prior package specification.
-- Passed both new Eurostat package validations and suite builds: 12 tax facts
-  and 9 ESSPROS facts, full lineage, and zero acceptance errors.
-- Extended Belgium and Eurostat regressions for FPB table counts/cells and
-  assertion boundary, vintage non-overlap, prior-output digests, Statbel pins,
-  and the declared 0.25% Statbel/FPB population comparison tolerance.
-- Passed 43 focused tests and the full merged-bundle regression: 157,177 facts,
-  148 packages, zero aggregate-key duplicates, and expected goldens throughout.
-- Recorded pins, counts, boundary evidence, curator commands, validation tails,
-  and consumer fact families in `LANE_C5_REPORT.md`.
-- Passed independent `ledger-source-fidelity` and `ledger-boundary` reviews with
-  no required corrections.
+- Checked for another `ots-anchor-main` process; this sandbox denies process-list
+  access to both `pgrep` and `ps`, so no process result was available.
+- Attempted the required `git fetch origin`; DNS resolution for GitHub failed.
+- Confirmed local `HEAD` and `origin/ots-anchor-main` both resolve to
+  `03b27674028a0d8cf5bdc71437f5944661f3c6cb`, then reset the worktree to that
+  remote-tracking ref.
+- Read the inherited progress log. Attempted to read PR #229 with `gh`; the same
+  GitHub DNS failure prevented access, so the supplied dispatch facts and PR-body
+  requirements are the current source of truth.
+- Accepted the Chronicle boundary and the prohibitions on changing `releases/`,
+  `ledger/`, or proof bytes.
+- Fixed proof classification to inspect local `ots info` structure first, give
+  any Bitcoin block-header attestation precedence over leftover pending
+  attestations, and then validate binding from the exact 0.7.2 output contract.
+- Replaced permissive verify-output recognition with exact/full-line parsing:
+  the exact mismatch line is the only mismatch, captured pending and paired
+  Bitcoin-disabled/manual lines bind successfully, and unknown output fails
+  closed.
+- Updated the hermetic fake client with the captured four-calendar pending,
+  mixed complete, mismatch, upgrade, and unrecognized outputs. All 17 focused
+  tests pass; focused Ruff lint and formatting checks pass.
+- Ran real status with the cached 0.7.2 executable. All 15 complete proofs were
+  recognized, then calendar DNS errors prevented binding classification for the
+  five pending proofs; the fixture suite covers their supplied captured output.
+- Replaced bot-branch import, PR creation, and auto-merge machinery with a
+  proof-only, non-force `git push origin HEAD:main` using only `contents: write`.
+- Added best-effort start-of-job logging for the three supplied effective-rules
+  and classic-protection API endpoints; metadata read failures warn but do not
+  block the publication attempt.
+- Retained the credential-free journal checkout, `run` + `verify` + `guard`,
+  `ots/`-only staging and committed-scope checks, and dirty-worktree refusal.
+- Added a three-attempt non-fast-forward path that fetches and rebases current
+  `main`, refreshes the journal, reruns all checks, and recommits before retry.
+  Failed-push classification uses fetched commit ancestry, covering server-side
+  ref races and ambiguous successes without parsing Git's localized output.
+- Updated both proof documentation sections to say automation pushes proof-only
+  commits to `main` and to document bounded retries plus the guarantees against
+  force-pushing or pushing the journal branch. `actionlint` passes.
+- Reconfirmed all 20 `.ots` blob bytes are unchanged (combined checksum
+  `420c3f54b47df5c58c58edc7202f580dcbad40193a4264bec20c133b71a375b1`)
+  and there are no `releases/` or `ledger/` changes.
+- Ran the final required suite: 17 focused tests, repository-wide Ruff lint,
+  focused Ruff formatting, and actionlint all pass.
+- Confirmed local proof structure with the real cached OpenTimestamps 0.7.2
+  executable: 15 Bitcoin-complete and 5 pending. Calendar DNS prevents the full
+  status binding pass, so the captured-output fixture tests are the documented
+  no-network fallback.
+- Wrote the final implementation, verification, integrity, and delivery report
+  to `out.md`.
+- Read PR #229 through the authenticated connector and prepared the replacement
+  “Publication path” body at `/tmp/pr229-body.md`. The exact `gh pr edit` command
+  failed on DNS, and the connector write was cancelled.
 
 ## Next
 
-- None; ready for handoff. No push was performed.
+- When GitHub access is available, push `ots-anchor-main` and run
+  `gh pr edit 229 --body-file /tmp/pr229-body.md` without merging the PR.

README.md

@@ -569,3 +569,14 @@ normalized_fact = convert_units(fact, 1000, "count")
   target selection, and calibration execution.
 - [thesis](https://github.com/PolicyEngine/thesis) - Public-facing official
   observations and analysis surfaces backed by Chronicle facts.
+
+## Bitcoin checkpoints for the witnessed journal
+
+The `codex/thesis-ledger-facts` branch's witnessed release manifests are
+additionally anchored through OpenTimestamps. Trusted automation pushes
+proof-only commits to `main`, where the mutable `ots/<stem>.json.ots` proofs
+live, while the immutable manifests and journal remain on their protected
+branch. Each proof binds a manifest's exact bytes into Bitcoin, giving the
+journal state it commits to an external anteriority bound. See
+[`ots/README.md`](ots/README.md) for the limits, cross-branch verification
+command, and publication design.

ots/README.md

@@ -0,0 +1,75 @@
+# Bitcoin checkpoints for the witnessed journal
+
+This directory contains [OpenTimestamps](https://opentimestamps.org) proofs for
+release manifests on the `codex/thesis-ledger-facts` journal branch. For a stem
+`<stem>`, `ots/<stem>.json.ots` commits to the exact bytes of
+`releases/manifests/<stem>.json` in a journal checkout. Those are the same bytes
+witnessed by the release's two RFC 3161 authorities and signed by its pinned
+producer key.
+
+## What a proof establishes
+
+A proof with a Bitcoin block attestation establishes that the manifest bytes
+existed no later than that block. Each manifest commits to the full journal
+bytes (`state.jsonlSha256` and `state.lineCount`), the immutable prefix, and the
+previous manifest. An attestation therefore bounds the existence time of that
+journal state and the manifest chain it incorporates without trusting this
+repository's Git history as the only checkpoint.
+
+The proof does not establish that the manifest's claims are true, that GitHub
+accepted a proposal at a particular time, or that no parallel fork exists. A
+rewritten history can acquire new anchors, but Bitcoin exposes the later time at
+which those replacement bytes first existed; it cannot be backdated.
+
+The 15 proofs initially carried here bind releases 0000–0014 and were first
+stamped on 2026-08-19. The daily workflow stamps later manifests after they
+appear and upgrades locally pending proof files when calendar attestations can
+be folded into the serialized proof.
+
+## Verify
+
+Check a proof against a real checkout of the journal branch:
+
+```console
+ots --no-bitcoin verify \
+  -f <journal-checkout>/releases/manifests/<stem>.json \
+  ots/<stem>.json.ots
+```
+
+The workflow pins the client as
+`uvx --from opentimestamps-client==0.7.2 ots`; use that invocation in place of
+`ots` for the same reproducible client version. `--no-bitcoin` verifies that the
+proof commits to the manifest's exact bytes and, when available, prints a block
+height and merkle root for manual checking against a Bitcoin source you trust.
+
+To sweep every manifest in a journal checkout against this proof tree:
+
+```console
+python3 scripts/ots_anchor.py verify \
+  --manifests <journal-checkout>/releases/manifests \
+  --ots-bin "uvx --from opentimestamps-client==0.7.2 ots"
+```
+
+This is strict: a mismatched or missing proof fails. Add `--require-bitcoin` to
+also fail while any committed proof file still contains only pending calendar
+attestations. `status` distinguishes that local serialized state from an
+attestation a calendar may resolve in memory during verification.
+
+## Why proofs and automation live on `main`
+
+The journal's `releases/` history is immutable, and its append gate admits only
+complete release bundles. OpenTimestamps proofs are operational artifacts:
+stamping creates them after a release exists, and upgrading rewrites them as
+calendar transactions confirm. They therefore cannot live under `releases/`.
+
+Keeping the proof tree, anchoring script, tests, and scheduled workflow together
+on protected `main` also establishes the privilege boundary. The workflow runs
+`main`'s trusted script against a separate, shallow journal checkout that has no
+persisted credential. It runs `run`, `verify`, and `guard`, stages only `ots/`,
+and refuses a dirty worktree or a committed path outside `ots/` before it pushes
+the proof-only commit directly to `main` with `git push origin HEAD:main`.
+
+A non-fast-forward rejection starts a bounded retry: fetch and rebase onto the
+new `origin/main`, refresh the credential-free journal checkout, rerun `run`,
+`verify`, and `guard`, then recommit and retry. The workflow makes at most three
+non-force push attempts and never pushes the journal branch.