diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fd6d616..552fbaac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,14 @@ jobs: run: > uv run pytest -q + # Keep this security-sensitive operational surface explicit even though + # the broad Chronicle lint and test commands above also cover it. + - name: Lint OpenTimestamps anchoring + run: uv run ruff check scripts/ots_anchor.py tests/test_ots_anchor.py + + - name: Test OpenTimestamps anchoring + run: uv run pytest -q tests/test_ots_anchor.py + - name: Build source input database run: | uv run chronicle --db /tmp/chronicle-targets-ci.db init diff --git a/.github/workflows/ots-anchor.yml b/.github/workflows/ots-anchor.yml new file mode 100644 index 00000000..c2560145 --- /dev/null +++ b/.github/workflows/ots-anchor.yml @@ -0,0 +1,176 @@ +name: OTS anchor + +# The trusted tool and mutable proofs live on main. The journal checkout is +# credential-free input only; no code from it is executed and it is never pushed. + +on: + schedule: + - cron: "23 6 * * *" + workflow_dispatch: + +permissions: {} + +concurrency: + group: ots-anchor-main + cancel-in-progress: false + +jobs: + anchor: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + # API evidence on 2026-09-02 showed no effective rules, required checks, + # reviews, or push restrictions blocking a GITHUB_TOKEN update to main. + contents: write + env: + MAIN_BRANCH: main + JOURNAL_BRANCH: codex/thesis-ledger-facts + OTS_BIN: uvx --from opentimestamps-client==0.7.2 ots + steps: + - name: Log main publication rule evidence + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + + log_api() { + endpoint="$1" + printf 'gh api %s\n' "$endpoint" + if ! gh api "$endpoint"; then + printf '::warning::Could not read %s; the bounded push remains authoritative.\n' \ + "$endpoint" >&2 + fi + } + + log_api "repos/$GITHUB_REPOSITORY/rules/branches/$MAIN_BRANCH" + log_api "repos/$GITHUB_REPOSITORY/rulesets" + log_api "repos/$GITHUB_REPOSITORY/branches/$MAIN_BRANCH/protection" + + - name: Check out trusted main + uses: actions/checkout@v4 + with: + ref: main + path: main + fetch-depth: 0 + persist-credentials: true + + - name: Check out journal manifests without credentials + uses: actions/checkout@v4 + with: + ref: codex/thesis-ledger-facts + path: journal + fetch-depth: 1 + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Anchor, verify, and publish proof-only changes + working-directory: main + shell: bash + run: | + set -euo pipefail + + manifest_dir="$GITHUB_WORKSPACE/journal/releases/manifests" + + anchor_and_verify() { + python3 scripts/ots_anchor.py run \ + --manifests "$manifest_dir" \ + --ots-bin "$OTS_BIN" + python3 scripts/ots_anchor.py verify \ + --manifests "$manifest_dir" \ + --ots-bin "$OTS_BIN" + python3 scripts/ots_anchor.py guard + } + + refresh_journal() { + git -C "$GITHUB_WORKSPACE/journal" fetch \ + --no-tags --depth=1 origin "$JOURNAL_BRANCH" + git -C "$GITHUB_WORKSPACE/journal" checkout \ + --detach FETCH_HEAD + } + + commit_proofs() { + git add -- ots/ + if git diff --cached --quiet; then + return + fi + if git diff --quiet "origin/$MAIN_BRANCH"...HEAD; then + git commit \ + -m "Update OpenTimestamps anchors for release manifests" + else + git commit --amend --no-edit + fi + } + + assert_commit_scope() { + outside=0 + while IFS= read -r changed_path; do + if [[ -z "$changed_path" ]]; then + continue + fi + case "$changed_path" in + ots/*) ;; + *) + printf 'refusing committed path outside ots/: %s\n' \ + "$changed_path" >&2 + outside=1 + ;; + esac + done < <(git diff --name-only "origin/$MAIN_BRANCH"...HEAD) + if [[ "$outside" -ne 0 ]]; then + return 1 + fi + if [[ -n "$(git status --porcelain)" ]]; then + git status --short >&2 + printf 'refusing to push a dirty worktree\n' >&2 + return 1 + fi + } + + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + + anchor_and_verify + commit_proofs + if git diff --quiet "origin/$MAIN_BRANCH"...HEAD; then + printf 'no proof changes to publish\n' + exit 0 + fi + assert_commit_scope + + for attempt in 1 2 3; do + if push_output="$(git push origin HEAD:main 2>&1)"; then + printf '%s\n' "$push_output" + exit 0 + fi + printf '%s\n' "$push_output" >&2 + if ! git fetch --no-tags origin main; then + printf 'push failed and main could not be fetched for retry analysis\n' >&2 + exit 1 + fi + if git merge-base --is-ancestor HEAD origin/main; then + printf 'local proof commit is already contained in main\n' + exit 0 + fi + if git merge-base --is-ancestor origin/main HEAD; then + printf 'push failed for a reason other than non-fast-forward\n' >&2 + exit 1 + fi + if [[ "$attempt" -eq 3 ]]; then + printf 'push remained non-fast-forward after 3 attempts\n' >&2 + exit 1 + fi + + git rebase origin/main + refresh_journal + anchor_and_verify + commit_proofs + if git diff --quiet origin/main...HEAD; then + printf 'proof changes already landed during retry\n' + exit 0 + fi + assert_commit_scope + done diff --git a/README.md b/README.md index 576919cd..c3f4da22 100644 --- a/README.md +++ b/README.md @@ -569,3 +569,14 @@ normalized_fact = convert_units(fact, 1000, "count") target selection, and calibration execution. - [thesis](https://github.com/PolicyEngine/thesis) - Public-facing official observations and analysis surfaces backed by Chronicle facts. + +## Bitcoin checkpoints for the witnessed journal + +The `codex/thesis-ledger-facts` branch's witnessed release manifests are +additionally anchored through OpenTimestamps. Trusted automation pushes +proof-only commits to `main`, where the mutable `ots/.json.ots` proofs +live, while the immutable manifests and journal remain on the journal +branch. Each proof binds a manifest's exact bytes into Bitcoin, giving the +journal state it commits to an external anteriority bound. See +[`ots/README.md`](ots/README.md) for the limits, cross-branch verification +command, and publication design. diff --git a/ots/0000-307cedbc91de43be.json.ots b/ots/0000-307cedbc91de43be.json.ots new file mode 100644 index 00000000..7a392f4e Binary files /dev/null and b/ots/0000-307cedbc91de43be.json.ots differ diff --git a/ots/0001-916626696d034b80.json.ots b/ots/0001-916626696d034b80.json.ots new file mode 100644 index 00000000..4afcca5d Binary files /dev/null and b/ots/0001-916626696d034b80.json.ots differ diff --git a/ots/0002-a69272175b73c83b.json.ots b/ots/0002-a69272175b73c83b.json.ots new file mode 100644 index 00000000..6ec331c3 Binary files /dev/null and b/ots/0002-a69272175b73c83b.json.ots differ diff --git a/ots/0003-cfae6e9b4524db6d.json.ots b/ots/0003-cfae6e9b4524db6d.json.ots new file mode 100644 index 00000000..fdfb7218 Binary files /dev/null and b/ots/0003-cfae6e9b4524db6d.json.ots differ diff --git a/ots/0004-36322993cf45b6d1.json.ots b/ots/0004-36322993cf45b6d1.json.ots new file mode 100644 index 00000000..fd906796 Binary files /dev/null and b/ots/0004-36322993cf45b6d1.json.ots differ diff --git a/ots/0005-9bcc4ff6b3fad5d2.json.ots b/ots/0005-9bcc4ff6b3fad5d2.json.ots new file mode 100644 index 00000000..f1025be9 Binary files /dev/null and b/ots/0005-9bcc4ff6b3fad5d2.json.ots differ diff --git a/ots/0006-770683e59da14f45.json.ots b/ots/0006-770683e59da14f45.json.ots new file mode 100644 index 00000000..65cae664 Binary files /dev/null and b/ots/0006-770683e59da14f45.json.ots differ diff --git a/ots/0007-2b5ed02908832f0c.json.ots b/ots/0007-2b5ed02908832f0c.json.ots new file mode 100644 index 00000000..6d8f4f0b Binary files /dev/null and b/ots/0007-2b5ed02908832f0c.json.ots differ diff --git a/ots/0008-070e797b855dce92.json.ots b/ots/0008-070e797b855dce92.json.ots new file mode 100644 index 00000000..525e5e2e Binary files /dev/null and b/ots/0008-070e797b855dce92.json.ots differ diff --git a/ots/0009-995768a31dd8fa6d.json.ots b/ots/0009-995768a31dd8fa6d.json.ots new file mode 100644 index 00000000..912f5887 Binary files /dev/null and b/ots/0009-995768a31dd8fa6d.json.ots differ diff --git a/ots/0010-6ba8c08f34189164.json.ots b/ots/0010-6ba8c08f34189164.json.ots new file mode 100644 index 00000000..3b565b04 Binary files /dev/null and b/ots/0010-6ba8c08f34189164.json.ots differ diff --git a/ots/0011-34319583df55ce83.json.ots b/ots/0011-34319583df55ce83.json.ots new file mode 100644 index 00000000..aac88721 Binary files /dev/null and b/ots/0011-34319583df55ce83.json.ots differ diff --git a/ots/0012-3a5ef7eeee484370.json.ots b/ots/0012-3a5ef7eeee484370.json.ots new file mode 100644 index 00000000..cbbec166 Binary files /dev/null and b/ots/0012-3a5ef7eeee484370.json.ots differ diff --git a/ots/0013-d47323bbaacda2d1.json.ots b/ots/0013-d47323bbaacda2d1.json.ots new file mode 100644 index 00000000..2ed1bb34 Binary files /dev/null and b/ots/0013-d47323bbaacda2d1.json.ots differ diff --git a/ots/0014-bd12e9e3e79a5529.json.ots b/ots/0014-bd12e9e3e79a5529.json.ots new file mode 100644 index 00000000..756e605f Binary files /dev/null and b/ots/0014-bd12e9e3e79a5529.json.ots differ diff --git a/ots/0015-fdcfd0e570214f6b.json.ots b/ots/0015-fdcfd0e570214f6b.json.ots new file mode 100644 index 00000000..bd5d740e Binary files /dev/null and b/ots/0015-fdcfd0e570214f6b.json.ots differ diff --git a/ots/0016-5226191699ae168d.json.ots b/ots/0016-5226191699ae168d.json.ots new file mode 100644 index 00000000..c549dbaf Binary files /dev/null and b/ots/0016-5226191699ae168d.json.ots differ diff --git a/ots/0017-efa7d60fece304f7.json.ots b/ots/0017-efa7d60fece304f7.json.ots new file mode 100644 index 00000000..55af7d53 Binary files /dev/null and b/ots/0017-efa7d60fece304f7.json.ots differ diff --git a/ots/0018-20974a5bdeeace01.json.ots b/ots/0018-20974a5bdeeace01.json.ots new file mode 100644 index 00000000..25c15982 Binary files /dev/null and b/ots/0018-20974a5bdeeace01.json.ots differ diff --git a/ots/0019-01d2f0bfb2ebff75.json.ots b/ots/0019-01d2f0bfb2ebff75.json.ots new file mode 100644 index 00000000..88dd57eb Binary files /dev/null and b/ots/0019-01d2f0bfb2ebff75.json.ots differ diff --git a/ots/README.md b/ots/README.md new file mode 100644 index 00000000..4e4a4a1a --- /dev/null +++ b/ots/README.md @@ -0,0 +1,109 @@ +# Bitcoin checkpoints for the witnessed journal + +This directory contains [OpenTimestamps](https://opentimestamps.org) proofs for +release manifests on the `codex/thesis-ledger-facts` journal branch. For a stem +``, `ots/.json.ots` commits to the exact bytes of +`releases/manifests/.json` in a journal checkout. Those are the same bytes +witnessed by the release's two RFC 3161 authorities and signed by its pinned +producer key. + +## What a proof establishes + +A proof with a Bitcoin block attestation establishes that the manifest bytes +existed no later than that block. Each manifest commits to the full journal +bytes (`state.jsonlSha256` and `state.lineCount`), the immutable prefix, and the +previous manifest. An attestation therefore bounds the existence time of that +journal state and the manifest chain it incorporates without trusting this +repository's Git history as the only checkpoint. + +The proof does not establish that the manifest's claims are true, that GitHub +accepted a proposal at a particular time, or that no parallel fork exists. A +rewritten history can acquire new anchors, but Bitcoin exposes the later time at +which those replacement bytes first existed; it cannot be backdated. + +As of 2026-09-02 this directory carries 20 proofs, one per release 0000–0019, +and every proof is committed whatever its state. The proofs for releases +0000–0014 were first stamped on 2026-08-19 and already contain Bitcoin block +attestations. The proofs for releases 0015–0019 were stamped on 2026-09-02 and +are committed while still pending: each holds only calendar commitments until +the workflow upgrades it in place. The daily workflow stamps later manifests +after they appear and upgrades pending proof files when calendar attestations +can be folded into the serialized proof. + +## Verify + +Check a proof against a real checkout of the journal branch: + +```console +ots --no-bitcoin verify \ + -f /releases/manifests/.json \ + ots/.json.ots +``` + +The workflow pins the client as +`uvx --from opentimestamps-client==0.7.2 ots`; use that invocation in place of +`ots` for the same reproducible client version. `--no-bitcoin` verifies that the +proof commits to the manifest's exact bytes and, when available, prints a block +height and merkle root for manual checking against a Bitcoin source you trust. + +To sweep every manifest in a journal checkout against this proof tree: + +```console +python3 scripts/ots_anchor.py verify \ + --manifests /releases/manifests \ + --ots-bin "uvx --from opentimestamps-client==0.7.2 ots" +``` + +This is strict: a mismatched or missing proof fails, and every manifest is +reported rather than only the first failure. Add `--require-bitcoin` to also +fail while any committed proof file still contains only pending calendar +attestations. `status` distinguishes that local serialized state from an +attestation a calendar may resolve in memory during verification. + +The script establishes the binding without calendar traffic: the +`File sha256 hash` that `ots info` reads out of the proof must equal the +manifest's SHA-256. It then runs the client's own `--no-bitcoin verify` as an +independent check and echoes that output into the log. The client's verify path +first asks each calendar for upgrades, so for a pending proof the log carries +lines such as `Got 1 attestation(s) from `, `Calendar : Pending +confirmation in Bitcoin blockchain`, or `Calendar : ` after a +transient failure. None of that is interpreted. Only the client's exact +`File does not match original!` line, or an exit status the client never uses, +fails a proof. + +## Why proofs and automation live on `main` + +The journal's `releases/` history is immutable, and its append gate admits only +complete release bundles. OpenTimestamps proofs are operational artifacts: +stamping creates them after a release exists, and upgrading rewrites them as +calendar transactions confirm. They therefore cannot live under `releases/`. + +Keeping the proof tree, anchoring script, tests, and scheduled workflow together +on `main` keeps the trusted code and the mutable proofs in one place. The +workflow runs `main`'s script against a separate, shallow journal checkout that +has no persisted credential. It runs `run`, `verify`, and `guard`, stages only +`ots/`, and refuses a dirty worktree or a committed path outside `ots/` before +it pushes the proof-only commit directly to `main` with a plain, non-force +`git push origin HEAD:main`. + +A non-fast-forward rejection starts a bounded retry: fetch and rebase onto the +new `origin/main`, refresh the credential-free journal checkout, rerun `run`, +`verify`, and `guard`, then recommit and retry. The workflow makes at most three +non-force push attempts and never pushes the journal branch. + +## What the `ots/`-only guard does and does not provide + +`main` is not protected. When the workflow was written (2026-09-02) the +repository API reported no rulesets, required checks, review requirements, or +push restrictions on `main`, and the direct push depends on that absence. The +workflow logs the current rule evidence at the start of every run, so a later +change shows up in the job log. + +The `guard` subcommand and the workflow's `assert_commit_scope` run client-side, +in the same job that holds the `contents: write` token. They bound what a +correctly functioning run can publish: a stray file, an unexpected edit, or a +dirty worktree stops the push. They are not a server-side control. A compromised +job, a malicious dependency pulled in by the pinned client, or anyone else with +push access to `main` is not constrained by them. A repository ruleset that +protects `main` and lists this workflow as a bypass actor would add the +server-side boundary this design does not provide. diff --git a/scripts/ots_anchor.py b/scripts/ots_anchor.py new file mode 100644 index 00000000..fcd3fa79 --- /dev/null +++ b/scripts/ots_anchor.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +"""Anchor witnessed release manifests in Bitcoin via OpenTimestamps. + +Each supplied release manifest is already witnessed by two RFC 3161 +authorities and a pinned producer signature over its exact bytes. This tool +adds an operator-independent witness: an OpenTimestamps proof over those same +exact bytes, committed as ``ots/.json.ots`` in this repository. Because +manifest ``state.jsonlSha256`` covers the full journal bytes and +``previousManifestSha256`` chains every earlier manifest, a Bitcoin +attestation over one manifest bounds the existence time of the whole journal +state it commits to. + +Proofs live in this repository's top-level ``ots/`` directory, never beside +the manifests. The manifests may be supplied from a separate, credential-free +journal checkout with ``--manifests``. OpenTimestamps upgrades rewrite proof +files in place, while the journal's release history is immutable. + +Subcommands: + +- ``run``: stamp any manifest that lacks a proof, then try to upgrade pending + proofs to complete Bitcoin attestations. Idempotent; safe on a schedule. +- ``verify``: check every proof against its manifest's current bytes and + report the state stored in each local proof. Every manifest is reported; + the exit status is nonzero if any proof is missing or mismatched. +- ``status``: list proofs and whether each is unanchored, mismatched, pending + locally, or Bitcoin-complete locally. Exits nonzero only if the client + could not inspect a proof, so the listing would be incomplete. +- ``guard``: fail if the repository has any change outside ``ots/``. + +Requires the ``ots`` CLI (PyPI ``opentimestamps-client``); stamping and +upgrading contact public calendar servers. The binding between a proof and a +manifest is established locally: the ``File sha256 hash`` that ``ots info`` +reads out of the proof must equal the manifest's SHA-256. The client's own +``--no-bitcoin verify`` then runs as an independent check. Its calendar +messages are logged, never parsed, and only its exact digest-mismatch line +can fail a proof. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import pathlib +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from typing import NamedTuple + +ROOT = pathlib.Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST_DIR = pathlib.Path("releases/manifests") +OTS_DIR = pathlib.Path("ots") +MANIFEST_NAME_RE = re.compile(r"^(\d{4})-([0-9a-f]{16})\.json$") +SUBPROCESS_TIMEOUT = 300 + +# Full-line output of opentimestamps-client 0.7.2 (otsclient/cmds.py). Calendar +# URLs and error reasons are untrusted text, so every match below is anchored +# to a whole line and nothing is matched inside text a calendar can influence. +# +# verify_command: logging.error("File does not match original!"), then exit 1. +_MISMATCH_LINE = "File does not match original!" +# info_command: print("File %s hash: %s" % (hash name, hex digest)) on stdout. +_INFO_FILE_HASH_LINE_RE = re.compile( + r"^File (?P[a-z0-9]+) hash: (?P[0-9a-fA-F]+)$", re.MULTILINE +) +_UPGRADE_PENDING_LINE_RE = re.compile( + r"^\s*(?:Failed!\s*)?Timestamp not complete\.?\s*$", re.MULTILINE +) +_INFO_BITCOIN_LINE_RE = re.compile( + r"^\s*verify BitcoinBlockHeaderAttestation\(\d+\)\s*$", re.MULTILINE +) +_INFO_PENDING_LINE_RE = re.compile( + r"^\s*verify PendingAttestation\([^\r\n]*\)\s*$", re.MULTILINE +) + + +class AnchorError(RuntimeError): + """A condition that must stop the anchoring run.""" + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(65536), b""): + digest.update(block) + return digest.hexdigest() + + +def discover_manifests(directory: pathlib.Path) -> list[pathlib.Path]: + if not directory.is_dir(): + raise AnchorError(f"manifest directory missing: {directory}") + manifests: list[pathlib.Path] = [] + for candidate in sorted(directory.iterdir()): + if not MANIFEST_NAME_RE.match(candidate.name): + continue + if candidate.is_symlink() or not candidate.is_file(): + raise AnchorError(f"manifest is not a regular file: {candidate}") + manifests.append(candidate) + if not manifests: + raise AnchorError(f"no release manifests found in {directory}") + return manifests + + +def check_manifest_name_digest(manifest: pathlib.Path) -> str: + """Refuse to anchor bytes that contradict the manifest's own filename.""" + + match = MANIFEST_NAME_RE.match(manifest.name) + if match is None: # discover_manifests already filtered on the pattern + raise AnchorError(f"unexpected manifest filename: {manifest.name}") + digest = sha256_file(manifest) + if digest[:16] != match.group(2): + raise AnchorError( + f"manifest {manifest.name} bytes hash to {digest[:16]}..., " + "which contradicts the filename; refusing to anchor" + ) + return digest + + +def proof_path(root: pathlib.Path, manifest: pathlib.Path) -> pathlib.Path: + return root / OTS_DIR / f"{manifest.name}.ots" + + +def ensure_ots_directory(root: pathlib.Path) -> pathlib.Path: + directory = root / OTS_DIR + if directory.is_symlink(): + raise AnchorError(f"proof directory must not be a symlink: {directory}") + directory.mkdir(parents=True, exist_ok=True) + if not directory.is_dir(): + raise AnchorError(f"proof path is not a directory: {directory}") + return directory + + +def check_proof_destination(proof: pathlib.Path) -> None: + """Reject proof paths that could redirect writes outside ``ots/``.""" + + if proof.is_symlink(): + raise AnchorError(f"proof path must not be a symlink: {proof}") + if proof.exists() and not proof.is_file(): + raise AnchorError(f"proof path is not a regular file: {proof}") + + +def _run_ots( + ots_bin: list[str], arguments: list[str], *, timeout: int = SUBPROCESS_TIMEOUT +) -> subprocess.CompletedProcess[str]: + command = [*ots_bin, *arguments] + try: + return subprocess.run( + command, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except FileNotFoundError as exc: + raise AnchorError( + f"ots binary not found ({command[0]!r}); install " + "opentimestamps-client or pass --ots-bin" + ) from exc + except subprocess.TimeoutExpired as exc: + raise AnchorError(f"ots timed out: {' '.join(command)}") from exc + + +def stamp_manifest( + root: pathlib.Path, manifest: pathlib.Path, ots_bin: list[str] +) -> pathlib.Path: + """Stamp a temporary copy, installing output only in a real ``ots/``.""" + + directory = ensure_ots_directory(root) + destination = proof_path(root, manifest) + check_proof_destination(destination) + if destination.exists(): + raise AnchorError(f"refusing to replace existing proof: {destination}") + + # Keeping the temporary directory beneath ots/ makes os.replace atomic and + # guarantees that even the client's temporary output cannot reach releases/. + with tempfile.TemporaryDirectory(prefix=".ots-anchor-", dir=directory) as name: + working_copy = pathlib.Path(name) / manifest.name + shutil.copyfile(manifest, working_copy) + completed = _run_ots(ots_bin, ["stamp", str(working_copy)]) + produced = working_copy.with_name(working_copy.name + ".ots") + if completed.returncode != 0 or produced.is_symlink() or not produced.is_file(): + raise AnchorError( + f"ots stamp failed for {manifest.name}: " + f"{completed.stderr.strip() or completed.stdout.strip()}" + ) + check_proof_destination(destination) + os.replace(produced, destination) + return destination + + +class ProofInfo(NamedTuple): + """What ``ots info`` reads out of a local proof file.""" + + digest: str + """Lowercase hex SHA-256 of the file the proof commits to.""" + state: str + """``"bitcoin"`` or ``"pending"``.""" + + +def inspect_proof(proof: pathlib.Path, ots_bin: list[str]) -> ProofInfo: + """Read the bound file digest and attestation state from the proof itself. + + ``ots info`` only deserializes the proof; it never contacts a calendar, so + it is the trusted source for both facts. The digest line must appear + exactly once and name SHA-256; anything else fails closed. + """ + + check_proof_destination(proof) + if not proof.is_file(): + raise AnchorError(f"proof file missing: {proof}") + completed = _run_ots(ots_bin, ["info", str(proof)]) + output = completed.stdout + completed.stderr + if completed.returncode != 0: + raise AnchorError(f"ots info failed for {proof.name}: {output.strip()}") + digest_lines = _INFO_FILE_HASH_LINE_RE.findall(completed.stdout) + if len(digest_lines) != 1: + raise AnchorError( + f"ots info for {proof.name} printed {len(digest_lines)} file hash " + "lines; expected exactly one" + ) + algorithm, digest = digest_lines[0] + if algorithm != "sha256" or len(digest) != 64: + raise AnchorError( + f"proof {proof.name} commits to a {algorithm} digest of " + f"{len(digest)} hex characters; only SHA-256 proofs are anchored" + ) + if _INFO_BITCOIN_LINE_RE.search(output): + state = "bitcoin" + elif _INFO_PENDING_LINE_RE.search(output): + state = "pending" + else: + raise AnchorError( + f"proof {proof.name} lists neither a Bitcoin nor a pending " + "attestation; refusing to guess" + ) + return ProofInfo(digest=digest.lower(), state=state) + + +def local_proof_state(proof: pathlib.Path, ots_bin: list[str]) -> str: + """Return the attestation state serialized in the local proof itself.""" + + return inspect_proof(proof, ots_bin).state + + +def proof_is_complete(proof: pathlib.Path, ots_bin: list[str]) -> bool: + """True only when the committed proof file carries a Bitcoin attestation.""" + + return local_proof_state(proof, ots_bin) == "bitcoin" + + +def _restore_upgrade_backup(proof: pathlib.Path, backup: pathlib.Path) -> None: + """Restore the client's backup after an interrupted or invalid upgrade.""" + + if not backup.exists() and not backup.is_symlink(): + return + if backup.is_symlink() or not backup.is_file(): + raise AnchorError(f"unsafe OpenTimestamps backup path: {backup}") + if proof.is_symlink(): + proof.unlink() + elif proof.exists(): + if not proof.is_file(): + raise AnchorError(f"cannot restore proof over non-file: {proof}") + proof.unlink() + os.replace(backup, proof) + + +def upgrade_proof( + manifest: pathlib.Path, proof: pathlib.Path, ots_bin: list[str] +) -> bool: + """Upgrade a pending proof and validate its replacement before cleanup. + + Returns whether the validated local replacement contains a Bitcoin block + attestation. A normal still-pending response restores the original proof. + """ + + check_proof_destination(proof) + if not proof.is_file(): + raise AnchorError(f"proof file missing: {proof}") + backup = proof.with_name(proof.name + ".bak") + if backup.exists() or backup.is_symlink(): + raise AnchorError(f"refusing to overwrite existing backup: {backup}") + + try: + completed = _run_ots(ots_bin, ["upgrade", str(proof)]) + except (AnchorError, OSError): + # A timeout can arrive after the client renamed the original proof. + _restore_upgrade_backup(proof, backup) + raise + output = completed.stdout + completed.stderr + if completed.returncode != 0: + _restore_upgrade_backup(proof, backup) + if _UPGRADE_PENDING_LINE_RE.search(output): + return False + raise AnchorError(f"ots upgrade failed for {proof.name}: {output.strip()}") + + try: + check_proof_destination(proof) + if not proof.is_file(): + raise AnchorError(f"ots upgrade removed proof: {proof.name}") + state = classify_proof(manifest, proof, ots_bin) + if state == "mismatch": + raise AnchorError( + f"upgraded proof {proof.name} does not match manifest bytes" + ) + except (AnchorError, OSError): + _restore_upgrade_backup(proof, backup) + raise + + if backup.exists() or backup.is_symlink(): + if backup.is_symlink() or not backup.is_file(): + _restore_upgrade_backup(proof, backup) + raise AnchorError(f"unsafe OpenTimestamps backup path: {backup}") + backup.unlink() + return state == "bitcoin" + + +def _log_client_output(label: str, completed: subprocess.CompletedProcess[str]) -> None: + """Echo client output into the run log without interpreting it. + + Every echoed line is indented so that untrusted calendar text can never + start a line: GitHub Actions reads ``::``-prefixed workflow commands from + job output. + """ + + print(f"{label}: exit status {completed.returncode}", file=sys.stderr) + for line in completed.stdout.splitlines() + completed.stderr.splitlines(): + print(f" {line}", file=sys.stderr) + + +def verify_proof_binding( + manifest: pathlib.Path, proof: pathlib.Path, ots_bin: list[str] +) -> bool: + """Run the client's own verification as an independent, fail-closed check. + + Returns false only when the client prints its exact digest-mismatch line, + which it does before consulting any calendar. Otherwise + ``--no-bitcoin verify`` exits 1 whether the proof is pending or complete, + and first asks each calendar for upgrades, printing lines such as + ``Got 1 attestation(s) from ``, ``Calendar : Pending + confirmation in Bitcoin blockchain``, or ``Calendar : `` + whose exact shape depends on calendar state and network conditions. That + output is logged verbatim and never parsed; the binding itself comes from + ``inspect_proof``. An exit status the client never uses means the + verification did not run, which fails closed. + """ + + completed = _run_ots( + ots_bin, ["--no-bitcoin", "verify", "-f", str(manifest), str(proof)] + ) + _log_client_output(f"ots verify {proof.name}", completed) + lines = completed.stdout.splitlines() + completed.stderr.splitlines() + if _MISMATCH_LINE in lines: + return False + if completed.returncode not in (0, 1): + raise AnchorError( + f"ots verify did not run to completion for {proof.name} " + f"(exit status {completed.returncode})" + ) + return True + + +def classify_proof( + manifest: pathlib.Path, proof: pathlib.Path, ots_bin: list[str] +) -> str: + """Classify a local proof as ``"mismatch"``, ``"pending"``, or ``"bitcoin"``. + + The digest ``ots info`` reads out of the proof must equal the manifest's + SHA-256. That comparison needs no calendar traffic and is the binding this + tool relies on; a mismatch is final and skips the client's verification. + Otherwise the client's own verification runs as a second, independent + check that can only downgrade the result. + """ + + info = inspect_proof(proof, ots_bin) + if info.digest != sha256_file(manifest): + return "mismatch" + if not verify_proof_binding(manifest, proof, ots_bin): + return "mismatch" + return info.state + + +def classify_manifest( + root: pathlib.Path, manifest: pathlib.Path, ots_bin: list[str] +) -> tuple[str, str]: + """Classify one manifest for a report without aborting the sweep. + + Returns ``(state, detail)``. ``state`` is ``"unanchored"``, ``"mismatch"``, + ``"error"``, ``"pending"``, or ``"bitcoin"``; ``detail`` explains the first + three. ``run`` deliberately does not use this: it must stop at the first + manifest it cannot anchor, while ``verify`` and ``status`` must report + every manifest so that no failure hides behind an earlier one. + """ + + try: + check_manifest_name_digest(manifest) + except AnchorError: + return "mismatch", "manifest bytes contradict its filename" + except OSError as exc: + return "error", str(exc) + proof = proof_path(root, manifest) + try: + check_proof_destination(proof) + if not proof.exists(): + return "unanchored", "no OpenTimestamps proof" + state = classify_proof(manifest, proof, ots_bin) + except (AnchorError, OSError) as exc: + return "error", str(exc) + if state == "mismatch": + return "mismatch", "proof does not match manifest bytes" + return state, "" + + +def command_run( + root: pathlib.Path, manifest_dir: pathlib.Path, ots_bin: list[str] +) -> int: + manifests = discover_manifests(manifest_dir) + ensure_ots_directory(root) + stamped: list[str] = [] + upgraded: list[str] = [] + pending: list[str] = [] + for manifest in manifests: + check_manifest_name_digest(manifest) + proof = proof_path(root, manifest) + check_proof_destination(proof) + if not proof.exists(): + stamp_manifest(root, manifest, ots_bin) + state = classify_proof(manifest, proof, ots_bin) + if state == "mismatch": + raise AnchorError( + f"new proof {proof.name} does not match manifest bytes" + ) + stamped.append(manifest.name) + if state == "pending": + pending.append(manifest.name) + continue + + # Binding is checked before completeness, including for a proof whose + # local structure already contains a Bitcoin attestation. + state = classify_proof(manifest, proof, ots_bin) + if state == "mismatch": + raise AnchorError( + f"proof {proof.name} does not match manifest bytes; refusing to skip" + ) + if state == "bitcoin": + continue + if upgrade_proof(manifest, proof, ots_bin): + upgraded.append(manifest.name) + else: + pending.append(manifest.name) + print( + f"ots anchor run: {len(manifests)} manifests, " + f"stamped {len(stamped)}, upgraded {len(upgraded)}, " + f"still pending {len(pending)}" + ) + for name in stamped: + print(f" stamped {name}") + for name in upgraded: + print(f" upgraded {name}") + return 0 + + +def command_verify( + root: pathlib.Path, + manifest_dir: pathlib.Path, + ots_bin: list[str], + *, + require_bitcoin: bool, +) -> int: + manifests = discover_manifests(manifest_dir) + failures: list[str] = [] + pending_count = 0 + bitcoin_count = 0 + for manifest in manifests: + state, detail = classify_manifest(root, manifest, ots_bin) + if state == "bitcoin": + bitcoin_count += 1 + elif state == "pending": + pending_count += 1 + if require_bitcoin: + failures.append(f"{manifest.name}: attestation not yet in local proof") + else: + failures.append(f"{manifest.name}: {detail}") + print( + f"ots anchor verify: {len(manifests)} manifests, " + f"{bitcoin_count} locally Bitcoin-complete, {pending_count} pending locally" + ) + for failure in failures: + print(f" FAIL {failure}", file=sys.stderr) + if failures: + return 1 + print("every release manifest has an OpenTimestamps proof bound to its exact bytes") + return 0 + + +def command_status( + root: pathlib.Path, manifest_dir: pathlib.Path, ots_bin: list[str] +) -> int: + manifests = discover_manifests(manifest_dir) + errors = 0 + for manifest in manifests: + state, detail = classify_manifest(root, manifest, ots_bin) + if state == "bitcoin": + label = "bitcoin attestation stored locally" + elif state == "pending": + label = "pending local proof" + elif state == "unanchored": + label = "unanchored" + elif state == "mismatch": + label = f"MISMATCH ({detail})" + else: + errors += 1 + label = f"ERROR ({detail})" + print(f"{manifest.name}: {label}") + # A mismatch is a finding this listing exists to show; a client failure + # means the listing is incomplete, which is the only nonzero exit here. + return 1 if errors else 0 + + +def git_changed_paths(root: pathlib.Path) -> list[str]: + """Return both sides of every porcelain-v1 change record.""" + + completed = subprocess.run( + ["git", "status", "--porcelain=v1", "-z", "--untracked-files=all"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise AnchorError(f"git status failed: {completed.stderr.strip()}") + fields = completed.stdout.split("\0") + paths: list[str] = [] + index = 0 + while index < len(fields): + record = fields[index] + if not record: + index += 1 + continue + if len(record) < 4 or record[2] != " ": + raise AnchorError(f"unrecognized git status record: {record!r}") + status = record[:2] + paths.append(record[3:]) + if "R" in status or "C" in status: + index += 1 + if index >= len(fields) or not fields[index]: + raise AnchorError("git status omitted a rename/copy source path") + paths.append(fields[index]) + index += 1 + return paths + + +def assert_only_ots_changes(root: pathlib.Path) -> int: + paths = git_changed_paths(root) + outside = sorted( + changed for changed in paths if not changed.startswith(f"{OTS_DIR.as_posix()}/") + ) + if outside: + details = ", ".join(outside) + raise AnchorError(f"refusing changes outside ots/: {details}") + return len(paths) + + +def command_guard(root: pathlib.Path) -> int: + changed_count = assert_only_ots_changes(root) + print(f"ots publication guard: {changed_count} changed path(s), all under ots/") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="anchor witnessed release manifests via OpenTimestamps" + ) + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--root", + type=pathlib.Path, + default=ROOT, + help=argparse.SUPPRESS, + ) + common.add_argument( + "--manifests", + type=pathlib.Path, + default=DEFAULT_MANIFEST_DIR, + help=( + "manifest directory, absolute or relative to --root (default: %(default)s)" + ), + ) + common.add_argument( + "--ots-bin", + default="ots", + help="ots invocation, shell-split (default: %(default)s)", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser( + "run", parents=[common], help="stamp missing proofs, upgrade pending" + ) + verify_parser = subparsers.add_parser( + "verify", + parents=[common], + help="check every proof against current manifest bytes", + ) + verify_parser.add_argument( + "--require-bitcoin", + action="store_true", + help="fail while any local proof still lacks a Bitcoin attestation", + ) + subparsers.add_parser( + "status", parents=[common], help="list proofs and their local state" + ) + subparsers.add_parser( + "guard", parents=[common], help="fail on repository changes outside ots/" + ) + args = parser.parse_args(argv) + + root = args.root.resolve() + manifest_dir = args.manifests + if not manifest_dir.is_absolute(): + manifest_dir = root / manifest_dir + manifest_dir = manifest_dir.resolve() + ots_bin = shlex.split(args.ots_bin) + try: + if not ots_bin: + raise AnchorError("--ots-bin cannot be empty") + if args.command == "run": + return command_run(root, manifest_dir, ots_bin) + if args.command == "verify": + return command_verify( + root, + manifest_dir, + ots_bin, + require_bitcoin=args.require_bitcoin, + ) + if args.command == "status": + return command_status(root, manifest_dir, ots_bin) + return command_guard(root) + except (AnchorError, OSError) as exc: + print(f"ots anchor failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_ots_anchor.py b/tests/test_ots_anchor.py new file mode 100644 index 00000000..a12c7bd2 --- /dev/null +++ b/tests/test_ots_anchor.py @@ -0,0 +1,835 @@ +"""Tests for scripts/ots_anchor.py. + +The suite never contacts calendar servers or Bitcoin: a fake ``ots`` +executable reproduces the opentimestamps-client 0.7.2 output contract +(stamp/upgrade/info/verify, as read from ``otsclient/cmds.py`` and observed +against the real client), and every invocation is logged so the tests can +assert which operations ran. Like the real client, the fake prints ``info`` +output on stdout and every logged message on stderr. +""" + +from __future__ import annotations + +import hashlib +import json +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + +sys.path.insert(0, str(ROOT / "scripts")) + +import ots_anchor # noqa: E402 + +CALENDARS = ( + "https://btc.calendar.catallaxy.com", + "https://finney.calendar.eternitywall.com", + "https://alice.btc.calendar.opentimestamps.org", + "https://bob.btc.calendar.opentimestamps.org", +) +BITCOIN_ATTESTATIONS = ( + (963242, "34ff137ec701d2ee72ac4f88a08ddee948932f6be2840a066198acfae077a24d"), + (963243, "583d3abcc52c06fdffa5ba3d24177b6fb6f048f63733e2f79734897648827278"), + (963253, "20f7fb6f9e04f098f4cdecb138618d62d4e302a22f1e144c1e3f16c7d97fb00e"), + (963257, "376fd236cf6f231bb0453fcb5b109d3dfc2bebfac2455f3e723f0a79b6919f75"), +) + + +def pending_line(calendar: str) -> str: + return f"Calendar {calendar}: Pending confirmation in Bitcoin blockchain\n" + + +def confirmed_line(calendar: str) -> str: + return f"Got 1 attestation(s) from {calendar}\n" + + +def manual_check_lines(block: int, merkle_root: str) -> str: + return ( + "Not checking Bitcoin attestation; Bitcoin disabled\n" + f"To verify manually, check that Bitcoin block {block} " + f"has merkleroot {merkle_root}\n" + ) + + +MISMATCH_VERIFY_OUTPUT = "File does not match original!\n" +PENDING_VERIFY_OUTPUT = "".join(pending_line(calendar) for calendar in CALENDARS) +COMPLETE_VERIFY_OUTPUT = "".join( + manual_check_lines(block, root) for block, root in BITCOIN_ATTESTATIONS +) +# A locally pending proof whose calendars have all confirmed: the client's +# verify path upgrades in memory first, then reports each attestation. +RESOLVED_VERIFY_OUTPUT = ( + "".join(confirmed_line(calendar) for calendar in CALENDARS) + COMPLETE_VERIFY_OUTPUT +) +# The normal progression: some calendars confirmed, the rest still pending. +MIXED_VERIFY_OUTPUT = ( + confirmed_line(CALENDARS[0]) + + pending_line(CALENDARS[1]) + + confirmed_line(CALENDARS[2]) + + pending_line(CALENDARS[3]) + + "".join( + manual_check_lines(block, root) for block, root in BITCOIN_ATTESTATIONS[:2] + ) +) +# Transient calendar failures are logged with the URLError reason text. +TRANSIENT_VERIFY_OUTPUT = ( + f"Calendar {CALENDARS[0]}: timed out\n" + f"Calendar {CALENDARS[1]}: [Errno -3] Temporary failure in name resolution\n" + + pending_line(CALENDARS[2]) + + pending_line(CALENDARS[3]) +) +PENDING_INFO_TREE = "".join( + f" verify PendingAttestation('{calendar}')\n" for calendar in CALENDARS +) +COMPLETE_INFO_TREE = """\ + verify PendingAttestation('https://btc.calendar.catallaxy.com') + verify BitcoinBlockHeaderAttestation(963257) + verify PendingAttestation('https://bob.btc.calendar.opentimestamps.org') + verify BitcoinBlockHeaderAttestation(963243) + verify PendingAttestation('https://alice.btc.calendar.opentimestamps.org') + verify BitcoinBlockHeaderAttestation(963242) + verify PendingAttestation('https://finney.calendar.eternitywall.com') + verify BitcoinBlockHeaderAttestation(963253) +""" + + +def info_output(digest: str, tree: str) -> str: + return f"File sha256 hash: {digest}\nTimestamp:\n{tree}" + + +FAKE_OTS = r""" +import hashlib +import json +import os +import pathlib +import sys + +LOG = pathlib.Path(os.environ["FAKE_OTS_LOG"]) +CALENDARS = ( + "https://btc.calendar.catallaxy.com", + "https://finney.calendar.eternitywall.com", + "https://alice.btc.calendar.opentimestamps.org", + "https://bob.btc.calendar.opentimestamps.org", +) +BITCOIN_ATTESTATIONS = ( + ( + 963242, + "34ff137ec701d2ee72ac4f88a08ddee948932f6be2840a066198acfae077a24d", + ), + ( + 963243, + "583d3abcc52c06fdffa5ba3d24177b6fb6f048f63733e2f79734897648827278", + ), + ( + 963253, + "20f7fb6f9e04f098f4cdecb138618d62d4e302a22f1e144c1e3f16c7d97fb00e", + ), + ( + 963257, + "376fd236cf6f231bb0453fcb5b109d3dfc2bebfac2455f3e723f0a79b6919f75", + ), +) + + +def log_line(message): + # otsclient/ots.py: logging.basicConfig(format="%(message)s") -> stderr. + print(message, file=sys.stderr) + + +def calendar_pending(calendar): + log_line(f"Calendar {calendar}: Pending confirmation in Bitcoin blockchain") + + +def calendar_confirmed(calendar): + log_line(f"Got 1 attestation(s) from {calendar}") + + +def manual_check(block, merkle_root): + log_line("Not checking Bitcoin attestation; Bitcoin disabled") + log_line( + f"To verify manually, check that Bitcoin block {block} " + f"has merkleroot {merkle_root}" + ) + + +def verify_pending_proof(mode): + # upgrade_timestamp() chatter first, then one report per attestation now + # held in memory, exactly as verify_timestamp() in otsclient/cmds.py does. + confirmed = 0 + if mode == "pending": + for calendar in CALENDARS: + calendar_pending(calendar) + elif mode == "resolved": + for calendar in CALENDARS: + calendar_confirmed(calendar) + confirmed = len(CALENDARS) + elif mode == "mixed": + calendar_confirmed(CALENDARS[0]) + calendar_pending(CALENDARS[1]) + calendar_confirmed(CALENDARS[2]) + calendar_pending(CALENDARS[3]) + confirmed = 2 + elif mode == "transient": + log_line(f"Calendar {CALENDARS[0]}: timed out") + log_line( + f"Calendar {CALENDARS[1]}: " + "[Errno -3] Temporary failure in name resolution" + ) + calendar_pending(CALENDARS[2]) + calendar_pending(CALENDARS[3]) + else: + raise SystemExit(f"unexpected FAKE_OTS_VERIFY_CALENDARS: {mode}") + for block, merkle_root in BITCOIN_ATTESTATIONS[:confirmed]: + manual_check(block, merkle_root) + + +def log(entry): + with LOG.open("a", encoding="utf-8") as handle: + handle.write(entry + "\n") + + +def read_proof(path): + return json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + + +def write_proof(path, payload): + pathlib.Path(path).write_text(json.dumps(payload), encoding="utf-8") + + +def main(): + arguments = [a for a in sys.argv[1:] if a != "--no-bitcoin"] + command = arguments[0] + log(command) + if command == "stamp": + target = pathlib.Path(arguments[1]) + digest = hashlib.sha256(target.read_bytes()).hexdigest() + write_proof( + str(target) + ".ots", {"digest": digest, "state": "pending"} + ) + for calendar in CALENDARS: + log_line(f"Submitting to remote calendar {calendar}") + return 0 + if command == "info": + proof = read_proof(arguments[1]) + spoof = os.environ.get("FAKE_OTS_INFO_SPOOF") + print(f"File sha256 hash: {proof['digest']}") + print("Timestamp:") + if spoof == "hash-line": + # A second full digest line must not be trusted over the first. + print(f"File sha256 hash: {os.environ['FAKE_OTS_SPOOF_DIGEST']}") + if proof["state"] == "bitcoin": + print(" verify PendingAttestation('https://btc.calendar.catallaxy.com')") + print(" verify BitcoinBlockHeaderAttestation(963257)") + print( + " verify PendingAttestation(" + "'https://bob.btc.calendar.opentimestamps.org')" + ) + print(" verify BitcoinBlockHeaderAttestation(963243)") + print( + " verify PendingAttestation(" + "'https://alice.btc.calendar.opentimestamps.org')" + ) + print(" verify BitcoinBlockHeaderAttestation(963242)") + print( + " verify PendingAttestation(" + "'https://finney.calendar.eternitywall.com')" + ) + print(" verify BitcoinBlockHeaderAttestation(963253)") + elif spoof == "attestation": + print( + "verify PendingAttestation(" + "'https://fake/BitcoinBlockHeaderAttestation(1)')" + ) + elif spoof == "hash-in-uri": + print( + " verify PendingAttestation('https://fake/" + f"File sha256 hash: {os.environ['FAKE_OTS_SPOOF_DIGEST']}')" + ) + else: + for calendar in CALENDARS: + print(f" verify PendingAttestation('{calendar}')") + return 0 + if command == "upgrade": + path = pathlib.Path(arguments[1]) + proof = read_proof(path) + upgrade = os.environ.get("FAKE_OTS_UPGRADE") + if upgrade == "success": + proof["state"] = "bitcoin" + write_proof(path, proof) + pathlib.Path(str(path) + ".bak").write_text( + "backup", encoding="utf-8" + ) + for calendar in CALENDARS: + calendar_confirmed(calendar) + log_line("Success! Timestamp complete") + return 0 + if upgrade == "broken": + path.replace(pathlib.Path(str(path) + ".bak")) + log_line("calendar response could not be serialized") + return 2 + for calendar in CALENDARS: + calendar_pending(calendar) + log_line("Failed! Timestamp not complete") + return 1 + if command == "verify": + target = pathlib.Path(arguments[arguments.index("-f") + 1]) + proof = read_proof(arguments[-1]) + digest = hashlib.sha256(target.read_bytes()).hexdigest() + if ( + digest != proof["digest"] + or os.environ.get("FAKE_OTS_VERIFY_FORCE_MISMATCH") == "yes" + ): + # otsclient/cmds.py verify_command: the digest check happens + # before any calendar is consulted. + log_line("File does not match original!") + return 1 + abnormal_exit = os.environ.get("FAKE_OTS_VERIFY_EXIT") + if abnormal_exit: + log_line("Traceback (most recent call last):") + log_line("RuntimeError: simulated client crash") + return int(abnormal_exit) + if proof["state"] == "bitcoin": + for block, merkle_root in BITCOIN_ATTESTATIONS: + manual_check(block, merkle_root) + return 1 + verify_pending_proof(os.environ.get("FAKE_OTS_VERIFY_CALENDARS", "pending")) + return 1 + raise SystemExit(f"unexpected fake ots command: {command}") + + +if __name__ == "__main__": + raise SystemExit(main()) +""" + + +def make_manifest(directory: Path, index: int, payload: bytes) -> Path: + digest = hashlib.sha256(payload).hexdigest() + path = directory / f"{index:04d}-{digest[:16]}.json" + path.write_bytes(payload) + return path + + +@pytest.fixture +def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict: + manifest_dir = tmp_path / "releases" / "manifests" + manifest_dir.mkdir(parents=True) + manifests = [ + make_manifest(manifest_dir, 0, b'{"releaseIndex": 0}\n'), + make_manifest(manifest_dir, 1, b'{"releaseIndex": 1}\n'), + ] + fake = tmp_path / "fake_ots.py" + fake.write_text(FAKE_OTS, encoding="utf-8") + log = tmp_path / "ots-invocations.log" + log.touch() + monkeypatch.setenv("FAKE_OTS_LOG", str(log)) + monkeypatch.setenv("FAKE_OTS_UPGRADE", "pending") + ots_bin = f"{shlex.quote(sys.executable)} {shlex.quote(str(fake))}" + return { + "root": tmp_path, + "manifest_dir": manifest_dir, + "manifests": manifests, + "ots_bin": ots_bin, + "log": log, + } + + +def run_cli(repo: dict, *arguments: str) -> int: + return ots_anchor.main( + [*arguments, "--root", str(repo["root"]), "--ots-bin", repo["ots_bin"]] + ) + + +def logged_commands(repo: dict) -> list[str]: + return repo["log"].read_text(encoding="utf-8").split() + + +def invoke_fake_ots(repo: dict, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [*shlex.split(repo["ots_bin"]), *arguments], + capture_output=True, + text=True, + check=False, + ) + + +def proof_for(repo: dict, index: int) -> Path: + return repo["root"] / "ots" / f"{repo['manifests'][index].name}.ots" + + +def rebind_proof(proof: Path, **changes: str) -> None: + """Rewrite a fake proof so it binds other bytes or carries another state.""" + + payload = json.loads(proof.read_text(encoding="utf-8")) + payload.update(changes) + proof.write_text(json.dumps(payload), encoding="utf-8") + + +def test_run_stamps_every_manifest_into_ots_dir(repo: dict) -> None: + assert run_cli(repo, "run") == 0 + proofs = sorted((repo["root"] / "ots").iterdir()) + assert [p.name for p in proofs] == [m.name + ".ots" for m in repo["manifests"]] + for manifest, proof in zip(repo["manifests"], proofs): + payload = json.loads(proof.read_text(encoding="utf-8")) + assert payload["digest"] == hashlib.sha256(manifest.read_bytes()).hexdigest() + + +def test_run_never_writes_into_releases(repo: dict) -> None: + before = sorted(p.name for p in repo["manifest_dir"].iterdir()) + assert run_cli(repo, "run") == 0 + after = sorted(p.name for p in repo["manifest_dir"].iterdir()) + assert before == after + + +def test_run_is_idempotent_and_upgrades_pending_proofs( + repo: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + assert run_cli(repo, "run") == 0 + assert logged_commands(repo).count("stamp") == 2 + + # Second run: calendars still pending — no new stamps, upgrade attempted. + assert run_cli(repo, "run") == 0 + assert logged_commands(repo).count("stamp") == 2 + assert logged_commands(repo).count("upgrade") == 2 + + # Third run: attestations land — proofs upgraded in place, .bak removed. + monkeypatch.setenv("FAKE_OTS_UPGRADE", "success") + assert run_cli(repo, "run") == 0 + for manifest in repo["manifests"]: + proof = repo["root"] / "ots" / f"{manifest.name}.ots" + assert json.loads(proof.read_text(encoding="utf-8"))["state"] == "bitcoin" + assert not proof.with_name(proof.name + ".bak").exists() + + # Fourth run: complete proofs are left untouched (no further upgrades). + upgrades_before = logged_commands(repo).count("upgrade") + assert run_cli(repo, "run") == 0 + assert logged_commands(repo).count("upgrade") == upgrades_before + + +def test_run_refuses_manifest_contradicting_its_filename(repo: dict) -> None: + rogue = repo["manifest_dir"] / f"0002-{'0' * 16}.json" + rogue.write_bytes(b'{"releaseIndex": 2}\n') + assert run_cli(repo, "run") == 1 + + +def test_verify_passes_with_pending_proofs_by_default(repo: dict) -> None: + assert run_cli(repo, "run") == 0 + assert run_cli(repo, "verify") == 0 + assert run_cli(repo, "verify", "--require-bitcoin") == 1 + + +def test_verify_fails_on_missing_proof(repo: dict) -> None: + assert run_cli(repo, "run") == 0 + proof_for(repo, 0).unlink() + assert run_cli(repo, "verify") == 1 + + +def test_verify_fails_when_proof_binds_different_bytes(repo: dict) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 1) + rebind_proof(proof, digest="ab" * 32) + assert ( + ots_anchor.classify_proof( + repo["manifests"][1], proof, shlex.split(repo["ots_bin"]) + ) + == "mismatch" + ) + assert run_cli(repo, "verify") == 1 + + +def test_fake_client_emits_the_real_mismatch_line(repo: dict) -> None: + """opentimestamps-client 0.7.2 logs ``File does not match original!``.""" + + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 1) + verify = invoke_fake_ots( + repo, "--no-bitcoin", "verify", "-f", str(repo["manifests"][0]), str(proof) + ) + assert verify.returncode == 1 + assert verify.stdout == "" + assert verify.stderr == MISMATCH_VERIFY_OUTPUT + assert verify.stderr.strip() == ots_anchor._MISMATCH_LINE + + +def test_verify_enumerates_every_mismatch(repo: dict, capsys) -> None: + assert run_cli(repo, "run") == 0 + for index in (0, 1): + rebind_proof(proof_for(repo, index), digest="ab" * 32) + capsys.readouterr() + + assert run_cli(repo, "verify") == 1 + captured = capsys.readouterr() + failures = [line for line in captured.err.splitlines() if "FAIL" in line] + assert failures == [ + f" FAIL {manifest.name}: proof does not match manifest bytes" + for manifest in repo["manifests"] + ] + assert "0 locally Bitcoin-complete, 0 pending locally" in captured.out + + +def test_verify_reports_tampered_manifest_without_aborting(repo: dict, capsys) -> None: + assert run_cli(repo, "run") == 0 + repo["manifests"][0].write_bytes(b'{"releaseIndex": 0, "tampered": true}\n') + capsys.readouterr() + + assert run_cli(repo, "verify") == 1 + captured = capsys.readouterr() + failures = [line for line in captured.err.splitlines() if "FAIL" in line] + assert failures == [ + f" FAIL {repo['manifests'][0].name}: manifest bytes contradict its filename" + ] + assert "0 locally Bitcoin-complete, 1 pending locally" in captured.out + + +def test_status_prints_mismatch_for_tampered_manifest_and_proof( + repo: dict, capsys +) -> None: + assert run_cli(repo, "run") == 0 + repo["manifests"][0].write_bytes(b'{"releaseIndex": 0, "tampered": true}\n') + rebind_proof(proof_for(repo, 1), digest="ab" * 32) + capsys.readouterr() + + assert run_cli(repo, "status") == 0 + assert capsys.readouterr().out.splitlines() == [ + f"{repo['manifests'][0].name}: MISMATCH (manifest bytes contradict its filename)", + f"{repo['manifests'][1].name}: MISMATCH (proof does not match manifest bytes)", + ] + + +def test_status_reports_multi_calendar_pending_proofs(repo: dict, capsys) -> None: + assert run_cli(repo, "status") == 0 + output = capsys.readouterr().out + assert output.count("unanchored") == 2 + + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + verify = invoke_fake_ots( + repo, + "--no-bitcoin", + "verify", + "-f", + str(repo["manifests"][0]), + str(proof), + ) + assert verify.returncode == 1 + assert verify.stdout == "" + assert verify.stderr == PENDING_VERIFY_OUTPUT + info = invoke_fake_ots(repo, "info", str(proof)) + assert info.returncode == 0 + digest = hashlib.sha256(repo["manifests"][0].read_bytes()).hexdigest() + assert info.stdout == info_output(digest, PENDING_INFO_TREE) + assert info.stderr == "" + upgrade = invoke_fake_ots(repo, "upgrade", str(proof)) + assert upgrade.returncode == 1 + assert upgrade.stderr == PENDING_VERIFY_OUTPUT + "Failed! Timestamp not complete\n" + + commands_before = len(logged_commands(repo)) + assert run_cli(repo, "status") == 0 + output = capsys.readouterr().out + assert output.count("pending local proof") == 2 + assert logged_commands(repo)[commands_before:] == ["info", "verify"] * 2 + + +def test_status_prefers_bitcoin_info_with_leftover_pending_attestations( + repo: dict, capsys +) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + rebind_proof(proof, state="bitcoin") + + info = invoke_fake_ots(repo, "info", str(proof)) + assert info.returncode == 0 + digest = hashlib.sha256(repo["manifests"][0].read_bytes()).hexdigest() + assert info.stdout == info_output(digest, COMPLETE_INFO_TREE) + verify = invoke_fake_ots( + repo, + "--no-bitcoin", + "verify", + "-f", + str(repo["manifests"][0]), + str(proof), + ) + assert verify.returncode == 1 + assert verify.stderr == COMPLETE_VERIFY_OUTPUT + + assert run_cli(repo, "status") == 0 + output = capsys.readouterr().out + assert output.count("bitcoin attestation stored locally") == 1 + assert output.count("pending local proof") == 1 + + +@pytest.mark.parametrize( + ("calendars", "expected_output", "logged_marker"), + [ + ("mixed", MIXED_VERIFY_OUTPUT, "Got 1 attestation(s) from"), + ("transient", TRANSIENT_VERIFY_OUTPUT, "Temporary failure in name resolution"), + ("resolved", RESOLVED_VERIFY_OUTPUT, "Got 1 attestation(s) from"), + ], + ids=["mixed", "transient", "resolved"], +) +def test_run_reaches_upgrade_despite_verify_calendar_chatter( + repo: dict, + monkeypatch: pytest.MonkeyPatch, + capsys, + calendars: str, + expected_output: str, + logged_marker: str, +) -> None: + """Calendar progress and transient errors during verify never abort a run. + + The real client's verify path upgrades in memory first, so a locally + pending proof whose calendars have confirmed prints ``Got N + attestation(s) from ...`` lines, and a calendar timeout prints + ``Calendar : ``. Neither is a grammar the tool checks: the + binding comes from ``ots info``, the chatter is logged, and the run goes + on to ``upgrade_proof`` for every pending proof. + """ + + assert run_cli(repo, "run") == 0 + monkeypatch.setenv("FAKE_OTS_VERIFY_CALENDARS", calendars) + proof = proof_for(repo, 0) + verify = invoke_fake_ots( + repo, "--no-bitcoin", "verify", "-f", str(repo["manifests"][0]), str(proof) + ) + assert verify.returncode == 1 + assert verify.stderr == expected_output + upgrades_before = logged_commands(repo).count("upgrade") + capsys.readouterr() + + assert run_cli(repo, "run") == 0 + captured = capsys.readouterr() + assert logged_commands(repo).count("upgrade") == upgrades_before + 2 + assert "still pending 2" in captured.out + assert f"ots verify {proof.name}: exit status 1" in captured.err + assert logged_marker in captured.err + + assert run_cli(repo, "status") == 0 + assert capsys.readouterr().out.count("pending local proof") == 2 + assert run_cli(repo, "verify") == 0 + + +def test_verify_chatter_is_indented_in_the_log( + repo: dict, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + """Echoed calendar text can never start a log line.""" + + assert run_cli(repo, "run") == 0 + monkeypatch.setenv("FAKE_OTS_VERIFY_CALENDARS", "transient") + capsys.readouterr() + + assert run_cli(repo, "status") == 0 + err_lines = capsys.readouterr().err.splitlines() + echoed = [line for line in err_lines if "Calendar " in line] + assert len(echoed) == 8 + assert all(line.startswith(" ") for line in echoed) + + +def test_classify_reports_mismatch_from_info_digest_without_calendar_traffic( + repo: dict, +) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 1) + rebind_proof(proof, digest="ab" * 32) + commands_before = len(logged_commands(repo)) + + assert ( + ots_anchor.classify_proof( + repo["manifests"][1], proof, shlex.split(repo["ots_bin"]) + ) + == "mismatch" + ) + assert logged_commands(repo)[commands_before:] == ["info"] + + +def test_classify_fails_closed_on_the_clients_mismatch_line( + repo: dict, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + """Even when ``ots info`` agrees, the client's own mismatch line wins.""" + + assert run_cli(repo, "run") == 0 + monkeypatch.setenv("FAKE_OTS_VERIFY_FORCE_MISMATCH", "yes") + proof = proof_for(repo, 0) + assert ( + ots_anchor.classify_proof( + repo["manifests"][0], proof, shlex.split(repo["ots_bin"]) + ) + == "mismatch" + ) + capsys.readouterr() + assert run_cli(repo, "verify") == 1 + assert capsys.readouterr().err.count("FAIL") == 2 + assert run_cli(repo, "run") == 1 + + +def test_classify_fails_closed_on_abnormal_verify_exit( + repo: dict, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + assert run_cli(repo, "run") == 0 + monkeypatch.setenv("FAKE_OTS_VERIFY_EXIT", "2") + capsys.readouterr() + + assert run_cli(repo, "status") == 1 + captured = capsys.readouterr() + assert captured.out.count("ERROR (ots verify did not run to completion") == 2 + assert run_cli(repo, "verify") == 1 + assert run_cli(repo, "run") == 1 + + +def test_inspect_proof_reads_digest_and_state(repo: dict) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + ots_bin = shlex.split(repo["ots_bin"]) + digest = hashlib.sha256(repo["manifests"][0].read_bytes()).hexdigest() + assert ots_anchor.inspect_proof(proof, ots_bin) == ots_anchor.ProofInfo( + digest=digest, state="pending" + ) + rebind_proof(proof, state="bitcoin", digest=digest.upper()) + assert ots_anchor.inspect_proof(proof, ots_bin) == ots_anchor.ProofInfo( + digest=digest, state="bitcoin" + ) + + +def test_inspect_proof_fails_closed_without_exactly_one_digest_line( + repo: dict, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + rebind_proof(proof, digest="ab" * 32) + digest = hashlib.sha256(repo["manifests"][0].read_bytes()).hexdigest() + monkeypatch.setenv("FAKE_OTS_INFO_SPOOF", "hash-line") + monkeypatch.setenv("FAKE_OTS_SPOOF_DIGEST", digest) + + with pytest.raises(ots_anchor.AnchorError, match="2 file hash lines"): + ots_anchor.inspect_proof(proof, shlex.split(repo["ots_bin"])) + capsys.readouterr() + assert run_cli(repo, "status") == 1 + assert "ERROR (ots info for" in capsys.readouterr().out + assert run_cli(repo, "run") == 1 + + +def test_inspect_proof_ignores_digest_text_inside_attestation_lines( + repo: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + rebind_proof(proof, digest="ab" * 32) + digest = hashlib.sha256(repo["manifests"][0].read_bytes()).hexdigest() + monkeypatch.setenv("FAKE_OTS_INFO_SPOOF", "hash-in-uri") + monkeypatch.setenv("FAKE_OTS_SPOOF_DIGEST", digest) + + ots_bin = shlex.split(repo["ots_bin"]) + assert ots_anchor.inspect_proof(proof, ots_bin).digest == "ab" * 32 + assert ots_anchor.classify_proof(repo["manifests"][0], proof, ots_bin) == "mismatch" + + +def test_manifests_option_reads_external_checkout(repo: dict) -> None: + external = repo["root"].parent / "journal" / "releases" / "manifests" + external.mkdir(parents=True) + manifest_names = [manifest.name for manifest in repo["manifests"]] + for manifest in repo["manifests"]: + manifest.replace(external / manifest.name) + + assert run_cli(repo, "run", "--manifests", str(external)) == 0 + assert sorted(proof.name for proof in (repo["root"] / "ots").iterdir()) == [ + f"{name}.ots" for name in manifest_names + ] + assert not list(external.glob("*.ots")) + + +def test_guard_accepts_only_ots_and_rejects_outside_changes(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + (tmp_path / "README.md").write_text("baseline\n", encoding="utf-8") + subprocess.run(["git", "-C", str(tmp_path), "add", "README.md"], check=True) + subprocess.run( + [ + "git", + "-C", + str(tmp_path), + "-c", + "user.name=OTS test", + "-c", + "user.email=ots@example.invalid", + "commit", + "-qm", + "baseline", + ], + check=True, + ) + (tmp_path / "ots").mkdir() + (tmp_path / "ots" / "proof.ots").write_text("proof", encoding="utf-8") + + assert ots_anchor.main(["guard", "--root", str(tmp_path)]) == 0 + (tmp_path / "README.md").write_text("changed\n", encoding="utf-8") + assert ots_anchor.main(["guard", "--root", str(tmp_path)]) == 1 + + +def test_run_reverifies_bitcoin_complete_proof_binding(repo: dict) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + rebind_proof(proof, state="bitcoin", digest="ab" * 32) + upgrades_before = logged_commands(repo).count("upgrade") + + assert run_cli(repo, "run") == 1 + assert logged_commands(repo).count("upgrade") == upgrades_before + + +def test_local_state_resists_calendar_output_spoofing( + repo: dict, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + assert run_cli(repo, "run") == 0 + monkeypatch.setenv("FAKE_OTS_INFO_SPOOF", "attestation") + monkeypatch.setenv("FAKE_OTS_VERIFY_CALENDARS", "resolved") + + assert run_cli(repo, "status") == 0 + assert capsys.readouterr().out.count("pending local proof") == 2 + assert run_cli(repo, "verify", "--require-bitcoin") == 1 + upgrades_before = logged_commands(repo).count("upgrade") + assert run_cli(repo, "run") == 0 + assert logged_commands(repo).count("upgrade") == upgrades_before + 2 + + +def test_failed_upgrade_restores_original_backup( + repo: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + original = proof.read_bytes() + monkeypatch.setenv("FAKE_OTS_UPGRADE", "broken") + + assert run_cli(repo, "run") == 1 + assert proof.read_bytes() == original + assert not proof.with_name(proof.name + ".bak").exists() + + +def test_timed_out_upgrade_restores_original_backup( + repo: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + assert run_cli(repo, "run") == 0 + proof = proof_for(repo, 0) + original = proof.read_bytes() + real_run_ots = ots_anchor._run_ots + + def time_out_after_backup(ots_bin, arguments, *, timeout=300): + if arguments[0] == "upgrade": + upgrade_target = Path(arguments[1]) + upgrade_target.replace( + upgrade_target.with_name(upgrade_target.name + ".bak") + ) + raise ots_anchor.AnchorError("ots timed out") + return real_run_ots(ots_bin, arguments, timeout=timeout) + + monkeypatch.setattr(ots_anchor, "_run_ots", time_out_after_backup) + assert run_cli(repo, "run") == 1 + assert proof.read_bytes() == original + assert not proof.with_name(proof.name + ".bak").exists() + + +def test_run_rejects_symlinked_proof_directory(repo: dict) -> None: + proof_directory = repo["root"] / "ots" + proof_directory.symlink_to(repo["manifest_dir"], target_is_directory=True) + + assert run_cli(repo, "run") == 1 + assert not list(repo["manifest_dir"].glob("*.ots")) diff --git a/tests/test_ots_anchor_real_client.py b/tests/test_ots_anchor_real_client.py new file mode 100644 index 00000000..57a8ec8f --- /dev/null +++ b/tests/test_ots_anchor_real_client.py @@ -0,0 +1,299 @@ +"""Exercise scripts/ots_anchor.py against the real opentimestamps-client. + +The fake client in test_ots_anchor.py encodes the client's output contract; +these tests check that contract against the client the workflow actually runs +(``uvx --from opentimestamps-client==0.7.2 ots``) without any network access: + +- ``ots info`` only deserializes the proof and never contacts a calendar. +- ``ots --no-bitcoin verify`` of mismatching bytes stops at the digest check, + before any calendar is consulted (otsclient/cmds.py, verify_command). +- The proofs built here carry either a Bitcoin block attestation, for which the + client's upgrade loop never runs, or a pending attestation naming a calendar + outside the client's default whitelist, which the client refuses to contact + (``Ignoring attestation from calendar ...: Calendar not in whitelist``). + +The client is tried offline first, then with uv's normal resolution, which +installs it on a CI runner. Set ``OTS_ANCHOR_TEST_OTS_BIN`` to use another +invocation. The module is skipped when no invocation works. +""" + +from __future__ import annotations + +import hashlib +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + +sys.path.insert(0, str(ROOT / "scripts")) + +import ots_anchor # noqa: E402 + +CLIENT_PIN = "opentimestamps-client==0.7.2" +DEFAULT_INVOCATIONS = ( + f"uvx --offline --from {CLIENT_PIN} ots", + f"uvx --from {CLIENT_PIN} ots", +) +CLIENT_TIMEOUT = 300 # seconds; the first online invocation installs the client + +# OpenTimestamps detached proof format (opentimestamps/core/timestamp.py, +# notary.py, serialize.py in opentimestamps 0.4.5): magic, version, file hash +# op tag, file digest, then the timestamp tree. The proofs built here attest +# the file digest directly, with no further operations. +HEADER_MAGIC = b"\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94" +MAJOR_VERSION = 1 +OP_SHA256_TAG = b"\x08" +PENDING_ATTESTATION_TAG = bytes.fromhex("83dfe30d2ef90c8e") +BITCOIN_ATTESTATION_TAG = bytes.fromhex("0588960d73d71901") +UNLISTED_CALENDAR = "https://calendar.example.invalid" +BLOCK_HEIGHT = 963242 +MISMATCH_LINE = "File does not match original!" + + +def varuint(value: int) -> bytes: + if value == 0: + return b"\x00" + encoded = bytearray() + while value: + septet = value & 0x7F + value >>= 7 + encoded.append(septet | 0x80 if value else septet) + return bytes(encoded) + + +def varbytes(payload: bytes) -> bytes: + return varuint(len(payload)) + payload + + +def pending_attestation(uri: str) -> bytes: + return PENDING_ATTESTATION_TAG + varbytes(varbytes(uri.encode("ascii"))) + + +def bitcoin_attestation(height: int) -> bytes: + return BITCOIN_ATTESTATION_TAG + varbytes(varuint(height)) + + +def build_proof(payload: bytes, *attestations: bytes) -> bytes: + """Serialize a detached proof that attests ``payload``'s SHA-256 directly.""" + + digest = hashlib.sha256(payload).digest() + tree = b"".join(b"\xff\x00" + attestation for attestation in attestations[:-1]) + tree += b"\x00" + attestations[-1] + return HEADER_MAGIC + varuint(MAJOR_VERSION) + OP_SHA256_TAG + digest + tree + + +def run_client( + invocation: list[str], *arguments: str +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [*invocation, *arguments], + capture_output=True, + text=True, + timeout=CLIENT_TIMEOUT, + check=False, + ) + + +@pytest.fixture(scope="module") +def real_ots_bin(tmp_path_factory: pytest.TempPathFactory) -> str: + probe = tmp_path_factory.mktemp("probe") / "probe.ots" + probe.write_bytes(build_proof(b"probe\n", pending_attestation(UNLISTED_CALENDAR))) + override = os.environ.get("OTS_ANCHOR_TEST_OTS_BIN") + invocations = (override,) if override else DEFAULT_INVOCATIONS + for invocation in invocations: + # --no-cache keeps the client from reading or writing ~/.cache/ots. + candidate = f"{invocation} --no-cache" + try: + completed = run_client(shlex.split(candidate), "info", str(probe)) + except (OSError, subprocess.TimeoutExpired): + continue + if completed.returncode == 0 and "File sha256 hash:" in completed.stdout: + return candidate + pytest.skip(f"{CLIENT_PIN} is not runnable here") + + +def anchor_root(tmp_path: Path) -> tuple[Path, Path]: + manifest_dir = tmp_path / "releases" / "manifests" + manifest_dir.mkdir(parents=True) + (tmp_path / "ots").mkdir() + return tmp_path, manifest_dir + + +def make_manifest(directory: Path, index: int, payload: bytes) -> Path: + digest = hashlib.sha256(payload).hexdigest() + path = directory / f"{index:04d}-{digest[:16]}.json" + path.write_bytes(payload) + return path + + +def write_proof(root: Path, manifest: Path, proof_bytes: bytes) -> Path: + proof = root / "ots" / f"{manifest.name}.ots" + proof.write_bytes(proof_bytes) + return proof + + +def run_cli(root: Path, ots_bin: str, *arguments: str) -> int: + return ots_anchor.main([*arguments, "--root", str(root), "--ots-bin", ots_bin]) + + +def test_committed_proofs_bind_the_digest_in_their_filenames( + real_ots_bin: str, +) -> None: + proofs = sorted((ROOT / "ots").glob("*.json.ots")) + assert proofs + for proof in proofs: + stem_digest = proof.name.split("-", 1)[1][:16] + info = ots_anchor.inspect_proof(proof, shlex.split(real_ots_bin)) + assert info.digest.startswith(stem_digest), proof.name + assert info.state in {"bitcoin", "pending"}, proof.name + + +def test_synthetic_proofs_round_trip_through_ots_info( + real_ots_bin: str, tmp_path: Path +) -> None: + payload = b'{"releaseIndex": 0}\n' + digest = hashlib.sha256(payload).hexdigest() + ots_bin = shlex.split(real_ots_bin) + + pending = tmp_path / "pending.ots" + pending.write_bytes(build_proof(payload, pending_attestation(UNLISTED_CALENDAR))) + assert ots_anchor.inspect_proof(pending, ots_bin) == ots_anchor.ProofInfo( + digest=digest, state="pending" + ) + + complete = tmp_path / "complete.ots" + complete.write_bytes( + build_proof( + payload, + bitcoin_attestation(BLOCK_HEIGHT), + pending_attestation(UNLISTED_CALENDAR), + ) + ) + assert ots_anchor.inspect_proof(complete, ots_bin) == ots_anchor.ProofInfo( + digest=digest, state="bitcoin" + ) + + +def test_client_prints_exact_mismatch_line_and_nothing_else( + real_ots_bin: str, tmp_path: Path +) -> None: + root, manifest_dir = anchor_root(tmp_path) + manifest = make_manifest(manifest_dir, 0, b'{"releaseIndex": 0}\n') + proof = write_proof( + root, + manifest, + build_proof(b"other bytes\n", pending_attestation(UNLISTED_CALENDAR)), + ) + + completed = run_client( + shlex.split(real_ots_bin), + "--no-bitcoin", + "verify", + "-f", + str(manifest), + str(proof), + ) + assert completed.returncode == 1 + assert completed.stdout == "" + assert completed.stderr == MISMATCH_LINE + "\n" + assert ots_anchor._MISMATCH_LINE == MISMATCH_LINE + assert ( + ots_anchor.verify_proof_binding(manifest, proof, shlex.split(real_ots_bin)) + is False + ) + + +def test_status_and_verify_enumerate_mismatches( + real_ots_bin: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root, manifest_dir = anchor_root(tmp_path) + good = make_manifest(manifest_dir, 0, b'{"releaseIndex": 0}\n') + write_proof( + root, good, build_proof(good.read_bytes(), bitcoin_attestation(BLOCK_HEIGHT)) + ) + swapped = make_manifest(manifest_dir, 1, b'{"releaseIndex": 1}\n') + write_proof( + root, + swapped, + build_proof(b'{"releaseIndex": 99}\n', pending_attestation(UNLISTED_CALENDAR)), + ) + tampered = make_manifest(manifest_dir, 2, b'{"releaseIndex": 2}\n') + write_proof( + root, + tampered, + build_proof(tampered.read_bytes(), pending_attestation(UNLISTED_CALENDAR)), + ) + tampered.write_bytes(b'{"releaseIndex": 2, "tampered": true}\n') + + assert run_cli(root, real_ots_bin, "status") == 0 + captured = capsys.readouterr() + assert captured.out.splitlines() == [ + f"{good.name}: bitcoin attestation stored locally", + f"{swapped.name}: MISMATCH (proof does not match manifest bytes)", + f"{tampered.name}: MISMATCH (manifest bytes contradict its filename)", + ] + assert " Not checking Bitcoin attestation; Bitcoin disabled" in captured.err + assert ( + f" To verify manually, check that Bitcoin block {BLOCK_HEIGHT} has merkleroot" + in captured.err + ) + + assert run_cli(root, real_ots_bin, "verify") == 1 + captured = capsys.readouterr() + failures = [line for line in captured.err.splitlines() if line.startswith(" FAIL")] + assert failures == [ + f" FAIL {swapped.name}: proof does not match manifest bytes", + f" FAIL {tampered.name}: manifest bytes contradict its filename", + ] + assert "3 manifests, 1 locally Bitcoin-complete, 0 pending locally" in captured.out + + +def test_run_reaches_upgrade_for_pending_proof( + real_ots_bin: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root, manifest_dir = anchor_root(tmp_path) + complete = make_manifest(manifest_dir, 0, b'{"releaseIndex": 0}\n') + write_proof( + root, + complete, + build_proof(complete.read_bytes(), bitcoin_attestation(BLOCK_HEIGHT)), + ) + pending = make_manifest(manifest_dir, 1, b'{"releaseIndex": 1}\n') + proof = write_proof( + root, + pending, + build_proof(pending.read_bytes(), pending_attestation(UNLISTED_CALENDAR)), + ) + original = proof.read_bytes() + + assert run_cli(root, real_ots_bin, "run") == 0 + captured = capsys.readouterr() + assert "2 manifests, stamped 0, upgraded 0, still pending 1" in captured.out + assert ( + f" Ignoring attestation from calendar {UNLISTED_CALENDAR}: " + "Calendar not in whitelist" + ) in captured.err + assert proof.read_bytes() == original + assert not proof.with_name(proof.name + ".bak").exists() + assert run_cli(root, real_ots_bin, "verify") == 0 + assert run_cli(root, real_ots_bin, "verify", "--require-bitcoin") == 1 + + +def test_run_refuses_proof_bound_to_other_bytes( + real_ots_bin: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root, manifest_dir = anchor_root(tmp_path) + manifest = make_manifest(manifest_dir, 0, b'{"releaseIndex": 0}\n') + proof = write_proof( + root, manifest, build_proof(b"other bytes\n", bitcoin_attestation(BLOCK_HEIGHT)) + ) + original = proof.read_bytes() + + assert run_cli(root, real_ots_bin, "run") == 1 + assert "does not match manifest bytes; refusing to skip" in capsys.readouterr().err + assert proof.read_bytes() == original diff --git a/tests/test_ots_anchor_workflow.py b/tests/test_ots_anchor_workflow.py new file mode 100644 index 00000000..92e95b54 --- /dev/null +++ b/tests/test_ots_anchor_workflow.py @@ -0,0 +1,101 @@ +"""Tests for the publication shell functions in .github/workflows/ots-anchor.yml.""" + +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "ots-anchor.yml" +PUBLISH_STEP = "Anchor, verify, and publish proof-only changes" +COMMIT_SUBJECT = "Update OpenTimestamps anchors for release manifests" +BOT_NAME = "github-actions[bot]" +BOT_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com" + + +def publish_script() -> str: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = workflow["jobs"]["anchor"]["steps"] + (step,) = [step for step in steps if step.get("name") == PUBLISH_STEP] + return step["run"] + + +def shell_function(script: str, name: str) -> str: + match = re.search( + rf"^{re.escape(name)}\(\) \{{\n.*?^\}}$", script, re.DOTALL | re.MULTILINE + ) + assert match is not None, f"{name}() not found in the publish step" + return match.group(0) + + +def git(repo: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *arguments], + check=True, + capture_output=True, + text=True, + ).stdout + + +def bot_checkout(tmp_path: Path) -> Path: + """A repository shaped like the job's main checkout after anchoring.""" + + repo = tmp_path / "main" + repo.mkdir() + git(repo, "init", "-q") + git(repo, "config", "user.name", BOT_NAME) + git(repo, "config", "user.email", BOT_EMAIL) + (repo / "README.md").write_text("baseline\n", encoding="utf-8") + (repo / "ots").mkdir() + (repo / "ots" / "0000.json.ots").write_bytes(b"pending") + git(repo, "add", "-A") + git(repo, "commit", "-qm", "baseline") + git(repo, "update-ref", "refs/remotes/origin/main", "HEAD") + return repo + + +def commit_proofs(repo: Path) -> None: + script = ( + "set -euo pipefail\n" + + shell_function(publish_script(), "commit_proofs") + + "\ncommit_proofs\n" + ) + subprocess.run( + ["bash", "-c", script], + cwd=repo, + env={**os.environ, "MAIN_BRANCH": "main"}, + check=True, + capture_output=True, + text=True, + ) + + +def test_publish_script_carries_no_coauthor_trailer() -> None: + assert "Co-Authored-By" not in publish_script() + + +def test_commit_proofs_writes_a_plain_bot_commit(tmp_path: Path) -> None: + repo = bot_checkout(tmp_path) + (repo / "ots" / "0000.json.ots").write_bytes(b"upgraded") + + commit_proofs(repo) + message = git(repo, "log", "-1", "--format=%B") + assert message.strip() == COMMIT_SUBJECT + assert "Co-Authored-By" not in message + assert git(repo, "log", "-1", "--format=%an <%ae>").strip() == ( + f"{BOT_NAME} <{BOT_EMAIL}>" + ) + + # A retry amends the proof-only commit instead of stacking a second one. + (repo / "ots" / "0001.json.ots").write_bytes(b"new") + commit_proofs(repo) + assert git(repo, "rev-list", "--count", "origin/main..HEAD").strip() == "1" + assert git(repo, "log", "-1", "--format=%B").strip() == COMMIT_SUBJECT + assert git(repo, "diff", "--name-only", "origin/main...HEAD").split() == [ + "ots/0000.json.ots", + "ots/0001.json.ots", + ]