From 51a98965cdaaffc97f16bd539fa19036ef987be7 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 14:03:16 +0200 Subject: [PATCH 1/5] Converge the annex plumbing: required=true always, pinned where PATH cannot serve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `filter=annex` makes the researcher's ordinary `git add` dispatch git-annex — through the filter drivers in `.git/config` and the four hooks `git annex init` writes, all resolved from *their* shell's PATH. When that PATH holds no git-annex (the engine driven through uvx puts nothing on it), the failure is the worst kind: git prints `git-annex: command not found`, exits 0, and stages the raw bytes into git history — a multi-gigabyte dataset in git proper, silently, on every clone forever. Measured, and now pinned by test_stock_plumbing_without_required_stages_raw_bytes_silently. Dataset convergence therefore owns the plumbing as one item (`annex-plumbing`), converged right after `git annex init`: - `filter.annex.required=true`, unconditionally. The same situation becomes git's own hard refusal (`fatal: … clean filter 'annex' failed`, exit 128, nothing staged), and the refusal's remedy is one `lc init`. - The filter drivers and hooks are pinned to the absolute path of the engine's bundled git-annex (the sibling of sys.executable) exactly where the shell's own PATH resolves none — probed with the engine's bin directory stripped out, since uvx and `uv run` front it on lc's environment and a naive which always finds the bundled copy. Where PATH does serve, the stock form is kept: it already works, and it outlives a prunable uv-cache path. - Repair only ever touches a broken state. A pin that runs is left alone even when ambient git-annex has appeared or the pin is not this engine's copy — both work, and rewriting would flip-flop between a tool install and a uvx cache. A pin that no longer runs is rewritten toward stock when the shell resolves git-annex, else toward the current engine's copy, and reports `repaired`; `required=true` makes the broken interval loud instead of corrupting. Hooks keep git-annex's own shebang and marker comment, which is also the ownership test — a hook without the marker is the user's and is never touched, by the write and the drift check alike. The stock spellings are mirrored from a real `git annex init`, never invented (test_stock_spellings_mirror_what_git_annex_itself_writes), so an upstream change fails the suite rather than repairing every project forever. `require_git_annex` stays a PATH check on purpose: it gates lc's *own* `git annex` subprocesses, which every install channel serves by fronting the bundled copy. Verified end to end: with the plumbing pinned and no git-annex on PATH, a plain `git add` stages an annex pointer and `git commit` runs the pinned pre-commit hook; a dead pin fails loudly, `lc init` reports it repaired, and the same add then succeeds. Every command and refusal quoted in the docs was executed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LsttSNGJ8kEMavA27ezW9L Signed-off-by: Francois Lanusse --- CLAUDE.md | 37 ++++- docs/api/dataset.md | 11 ++ docs/api/project.md | 14 +- docs/architecture.md | 8 ++ docs/cli/init.md | 26 +++- docs/user/troubleshooting.md | 30 ++++ src/lightcone/engine/dataset.py | 197 +++++++++++++++++++++++++ src/lightcone/engine/project.py | 122 +++++++++++++++- tests/conftest.py | 18 ++- tests/test_dataset.py | 142 ++++++++++++++++++ tests/test_project.py | 247 +++++++++++++++++++++++++++++++- 11 files changed, 835 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7c73800..f8d1a32 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -327,7 +327,7 @@ user owns: | `.python-version` | The exact patch of the interpreter `lc` is running on | | `uv.lock`, `.venv` | **Derived** — converged by correctness, not existence: `uv lock --check` / `uv sync --locked --exact --check` decide, then `uv lock` / `uv sync --locked --exact --compile-bytecode` repair | | `.gitignore` | One managed block of patterns; convergence ensures each is present | -| `.git` + the annex | `git init` then `git annex init` — results are versioned in the project's own repository | +| `.git` + the annex | `git init` then `git annex init` — results are versioned in the project's own repository — then the **annex plumbing** (`filter.annex.required=true` always; filter + hooks pinned to the bundled git-annex only where the shell's `PATH` resolves none — see the storage invariants) | | `.gitattributes` | The storage policy: what git-annex holds and what git carries. Line-managed, like `.gitignore` | | `.datalad/config` | A `datalad.dataset.id` UUID, generated once. Read back only by `dataset.dataset_id`, through `git config -f`, for the run record's `dsid` | | `data/` + `README.md` | Where declared inputs live; annexed, and committed before anything computes on them | @@ -342,8 +342,8 @@ user owns: Universes are discovered by `glob("*.yaml")`, which is empty-not-error on a missing directory. `tests/test_project.py::test_a_clone_of_a_converged_project_is_converged` pins this: a clone must need nothing but `.venv` and `git annex init`. - Those two are the exemptions, and for one reason — they are local state - git does not clone. + Those (with the annex plumbing) are the exemptions, and for one + reason — they are local state git does not clone. - **Convergence, not scaffolding.** Each item is created if missing, offered to a conservative `repair(text) -> str | None` hook otherwise, and left alone when the hook returns `None`. `--check` computes the @@ -545,6 +545,37 @@ for lc's subprocesses exactly as for their own git, and no different from which `git` itself runs. Nothing records the annex version anyway (see the Recorded decision on the engine's dependency closure). +**"By construction" covers every install channel — except the +researcher's own shell**, which is the one place `filter=annex` sends +git looking for git-annex (the filter config and the four hooks +`git annex init` writes, all `PATH`-resolved). Under `uvx` nothing +lands on the user's `PATH`, and a `git add` whose clean filter cannot +start prints an error, **exits 0, and stages the raw bytes into git +history** (measured, and pinned by +`test_stock_plumbing_without_required_stages_raw_bytes_silently`). So +dataset convergence owns the **annex plumbing** as an item +(`annex-plumbing`, 2026-08): `filter.annex.required=true` always — +git's own hard failure instead of silent corruption, with `lc init` as +the remedy — and the filter drivers plus hooks pinned to the absolute +path of the engine's bundled git-annex (`project.bundled_annex`, the +sibling of `sys.executable`) **only where the shell's own `PATH` +resolves no git-annex** (`project.ambient_annex`, probed with the +engine's bin stripped out). Where `PATH` serves, stock plumbing stays: +it already works and outlives any prunable engine path. Repair is +asymmetric by decision: a pin that runs is never rewritten, even when +it is not this engine's copy or ambient git-annex has appeared (both +work; rewriting flip-flops between a tool install and a uvx cache) — +only a broken state is rewritten, toward stock when the shell resolves +git-annex, else toward this engine's copy. Hooks keep git-annex's own +marker comment ("automatically configured by git-annex"), which is +also the ownership test — a hook without it is the user's and is never +touched. The stock spellings are mirrored, never invented: +`test_stock_spellings_mirror_what_git_annex_itself_writes` pins them +against a real `git annex init`. `require_git_annex` stays a `PATH` +check, deliberately — it gates lc's *own* `git annex` subprocesses, +which still dispatch from lc's environment, where every install +channel fronts the bundled copy. + **A project can sit inside a larger repository, so `dataset.status` is scoped and relativised.** `lc init subdir/` adopts an enclosing work tree rather than nesting a new one — that is a supported layout — and diff --git a/docs/api/dataset.md b/docs/api/dataset.md index 89c3dd1..50c07e0 100644 --- a/docs/api/dataset.md +++ b/docs/api/dataset.md @@ -20,6 +20,9 @@ Source: `src/lightcone/engine/dataset.py` (+ | `last_writer(root, dir)` | Who last touched an output's directory — the foreign-write question. Answers "cannot say" as empty, never an error. | | `require_committer(root)` | Refuses a repository with no git identity, before any recipe spends time. Asked as `git var`, the question a commit itself asks. | | `dataset_id(root)` | The DataLad dataset UUID, read via `git config -f`. | +| `converge_annex_plumbing(root, pin)` | Write the plumbing the researcher's own `git add` dispatches git-annex through: `required=true` always, filter drivers + four hooks in stock (`PATH`-resolved) or pinned (absolute-path) form. | +| `annex_plumbing_current(root, *, stock_ok)` | The no-write drift probe for the same, `lc init --check`'s question. | +| `pinned_annex(root)` / `annex_runs(path)` | The recorded pin, and whether it still runs — the repair rule's two reads. | ## What must stay true @@ -49,6 +52,14 @@ Source: `src/lightcone/engine/dataset.py` (+ - **Committing an archive or dot-named file needs `annex.dotfiles`** — git-annex routes dotfiles to git whatever `largefiles` says, and without the flag an image archive lands as a git blob, silently. +- **`filter.annex.required=true` is the storage policy's safety net.** + Without it, a `git add` whose shell cannot resolve git-annex prints + an error, exits 0, and stages the raw bytes into git history — + measured, and pinned by + `test_stock_plumbing_without_required_stages_raw_bytes_silently`. + The stock filter/hook spellings are mirrored from what + `git annex init` itself writes, never invented, and a hook without + git-annex's own marker comment is the user's — never rewritten. ## Tests diff --git a/docs/api/project.md b/docs/api/project.md index a8c9f02..2684576 100644 --- a/docs/api/project.md +++ b/docs/api/project.md @@ -17,6 +17,7 @@ Source: `src/lightcone/engine/project.py` (+ | `declared_project()` | The weaker question — what the repository carries, without `.venv`. One caller: the worker entry point, which builds the venv a moment later. | | `mode(root)` | `"direct"` or `"containerized"` — presence of `[tool.lightcone.image]`, nothing else. | | `uv_prefix(root, *, sync)` | The one spelling of the project uv hop. Callers differ only in `sync`: a probe converges the environment, a recipe must not. | +| `bundled_annex()` / `ambient_annex()` | The two git-annex questions: the engine's own copy (beside `sys.executable`) and the one the researcher's *shell* would resolve (`PATH` minus the engine's bin). The annex-plumbing item decides between stock and pinned form from the second, and pins to the first. | | `project_name(dir)` | PEP 503-ish name from the directory name. | | `_run` / `_check_call` | Every external tool invocation, and the suite's one monkeypatch point. | | `ProjectError` | The engine's one exception; the CLI translates it once. | @@ -38,9 +39,16 @@ Source: `src/lightcone/engine/project.py` (+ `.gitattributes` are converged entry-wise, order judged against the template). - **Only what git can carry is converged.** No `src/`, no empty - directories — a clone must need nothing but `.venv` and - `git annex init`, and - `test_a_clone_of_a_converged_project_is_converged` pins it. + directories — a clone must need nothing but `.venv`, `git annex + init` and the annex plumbing (all three local state git does not + clone), and `test_a_clone_of_a_converged_project_is_converged` pins + it. +- **The annex plumbing repairs only broken states.** + `filter.annex.required=true` always; the filter and hooks pinned to + the engine's git-annex only where the researcher's shell resolves + none; a working pin never rewritten (both forms work — rewriting + flip-flops between installs), a dead one rewritten toward stock when + the shell has git-annex, else toward this engine's copy. - **There is no discovery.** The invoked directory is the project or it is a clean error; every uv call carries an explicit `--project`. - **Templates are files** (`templates/files/*.tmpl`, `string.Template` diff --git a/docs/architecture.md b/docs/architecture.md index 6d28ba8..85da6c4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -104,6 +104,14 @@ git carries pointers and history, the annex carries bytes, and default; `data/` and `results/` opt out). A researcher only ever types ordinary `git add` / `git commit`. +That ordinary `git add` dispatches git-annex from the *researcher's* +`PATH`, and a shell that cannot resolve it silently stages raw bytes +into git history — so `lc init` also owns the **annex plumbing**: +`filter.annex.required=true` always (the same situation becomes git's +own hard failure, remedied by `lc init`), and the filter plus hooks +pinned to the engine's bundled git-annex by absolute path exactly +where the shell's `PATH` resolves none. + Each output is committed with a **run record** — a `[DATALAD RUNCMD]` commit message whose `cmd` reconstructs the engine (`uv run --no-project --with lightcone-cli==`) and re-executes the diff --git a/docs/cli/init.md b/docs/cli/init.md index e6429b7..39103ce 100644 --- a/docs/cli/init.md +++ b/docs/cli/init.md @@ -25,7 +25,8 @@ manages, and never overwrites files you own: - **Repaired** — derived artifacts that have drifted: a `uv.lock` that no longer matches `pyproject.toml`, a `.venv` that no longer matches the lock, a managed `.gitignore` or `.gitattributes` entry that a - newer `lc` added. Repairs only ever append or rebuild derived state; + newer `lc` added, annex plumbing that no longer resolves a working + git-annex. Repairs only ever append or rebuild derived state; hand-written lines are never reordered or removed. - **Blocked** — something convergence can see but must not fix by appending: a `.gitignore` rule that would silently swallow @@ -80,6 +81,29 @@ Two things it deliberately does *not* create: a `src/` directory directories), and any dependency in `pyproject.toml` — the lock carries only what *your* analysis imports, added with `uv add`. +Inside `.git`, convergence also owns the **annex plumbing** — the +filter configuration and hooks that let *your own* `git add` reach +git-annex (reported as the `annex-plumbing` item). Two guarantees: + +- `filter.annex.required=true`, always. Without it, a `git add` whose + shell cannot find git-annex prints an error, **exits 0, and stages + the raw bytes into git history** — a 2 GB dataset in git proper, on + every clone, forever. With it, the same situation is a hard, loud + failure and nothing is staged. +- Where your shell's `PATH` does not resolve `git-annex` (for example + when the engine was run through `uvx`, which puts nothing on your + `PATH`), the filter and hooks are pinned to the absolute path of the + git-annex that ships with `lc` itself, so the ordinary `git add` the + docs promise keeps working. Where your `PATH` does resolve one, the + stock, `PATH`-resolved form is kept — it already works, and it + outlives any particular `lc` install. + +A pinned path can die (a pruned `uv` cache, a moved install); thanks to +`required=true` that failure is loud, and the next `lc init` repairs +the pin. See +[`fatal: … clean filter 'annex' failed`](../user/troubleshooting.md#fatal-clean-filter-annex-failed) +in the troubleshooting guide. + ## Options | Option | Default | Effect | diff --git a/docs/user/troubleshooting.md b/docs/user/troubleshooting.md index e13edf6..d44c626 100644 --- a/docs/user/troubleshooting.md +++ b/docs/user/troubleshooting.md @@ -114,6 +114,36 @@ read-only verbs (`lc status`, `--check`) never transfer data, so they report the fact instead. Fetch by hand only when you want the bytes for your own inspection. +## "fatal: … clean filter 'annex' failed" + +``` +git-annex filter-process: line 1: git-annex: command not found +error: could not read greeting from subprocess 'git-annex filter-process' +error: initialization for subprocess 'git-annex filter-process' failed +fatal: data/catalog.fits: clean filter 'annex' failed +``` + +Your `git add` could not reach git-annex — either your shell's `PATH` +lost it, or the project's plumbing is pinned to an `lc` install that no +longer exists (a pruned `uv` cache, a moved install). The remedy is +one command, from the project root: + +```bash +lc init +``` + +It reports the plumbing `repaired` and re-points it at a git-annex +that actually runs — the one that ships with `lc` itself when your +`PATH` has none. + +This failure is deliberately loud. `lc init` sets +`filter.annex.required=true` in every project precisely because +without it git handles the same situation by printing the error, +**exiting 0, and staging your data's raw bytes into git history** — +committing a multi-gigabyte dataset into git proper, silently, where +every clone carries it forever. A refused `git add` costs you one +`lc init`; the silent version costs you the repository. + ## "… and this is a NERSC login node" `lc materialize` executes recipes, and on centers `lc` recognizes it diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index def066f..61f3be8 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -18,6 +18,8 @@ from __future__ import annotations +import os +import shlex from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -102,6 +104,201 @@ def ignore_rule(directory: Path, path: str) -> str | None: return proc.stdout.splitlines()[0].split("\t")[0] +# ============================================================================= +# The annex plumbing: how the researcher's own git reaches git-annex +# ============================================================================= +# +# `git annex init` wires the repository to a bare `git-annex`, resolved +# from whatever PATH the *researcher's* git runs under — the filter +# drivers in `.git/config` and four hooks. When their shell has no +# git-annex (lc itself always does — it bundles one), a plain `git add` +# prints an error, exits 0, and stages the raw bytes into git history: +# a 2 GB dataset lands in git proper, silently, on every clone forever. +# `filter.annex.required=true` turns that into git's own hard failure, +# and — only where PATH would not resolve git-annex — the plumbing is +# pinned to an absolute path instead, so the ordinary `git add` the docs +# promise keeps working. Measured: with the filter pinned, `git add` of +# annexed content stages a pointer with no git-annex on PATH at all. + + +#: The marker `git annex init` leaves in every hook it writes — and the +#: ownership test: git-annex itself rewrites only hooks that carry it, +#: and so does lc. A hook without it is the user's, never touched. +_HOOK_MARKER = "automatically configured by git-annex" + +#: What each hook runs — git-annex's own hook bodies, mirrored, with the +#: annex spelling left open: ``git annex`` in stock form, an absolute +#: path when pinned. +_HOOKS = { + "pre-commit": "{annex} pre-commit .", + "post-checkout": "{annex} smudge --update", + "post-merge": "{annex} smudge --update", + "post-receive": ( + "if {annex} post-receive --help >/dev/null 2>&1; then {annex} post-receive; fi" + ), +} + +#: The filter drivers `git annex init` configures, same convention +#: (stock spells these ``git-annex``, without the space). The +#: smudge/clean pair stays beside ``process``: git falls back to it when +#: the process filter cannot start, and a fallback that resolves +#: git-annex differently from the filter it stands in for would route +#: the same add two ways. +_FILTERS = { + "filter.annex.process": "{annex} filter-process", + "filter.annex.smudge": "{annex} smudge -- %f", + "filter.annex.clean": "{annex} smudge --clean -- %f", +} + + +def pinned_annex(directory: Path) -> Path | None: + """Read which absolute git-annex the filter is pinned to, if any. + + The record convergence's repair rule works from: the first word of + ``filter.annex.process``. Stock plumbing spells it bare + (``git-annex filter-process``), which is not a pin. + + Args: + directory: A directory inside the repository. + + Returns: + The pinned executable, or ``None`` for stock or absent plumbing. + """ + try: + words = shlex.split(_config(directory, "filter.annex.process")) + except ValueError: + return None + if words and Path(words[0]).is_absolute(): + return Path(words[0]) + return None + + +def annex_runs(path: Path) -> bool: + """Whether *path* is still an executable — the repair-rule probe. + + A pinned engine can be moved or its uv cache pruned; a pin that no + longer runs is what turns ``required=true``'s loud failure back into + a working ``git add`` on the next ``lc init``. + + Args: + path: The recorded git-annex executable. + + Returns: + True if the file exists and is executable. + """ + return path.is_file() and os.access(path, os.X_OK) + + +def converge_annex_plumbing(directory: Path, pin: Path | None) -> None: + """Write the plumbing that lets the researcher's own git find git-annex. + + ``filter.annex.required=true`` always: without it a ``git add`` whose + filter cannot start exits 0 and stages raw bytes into git history — + silent corruption; with it, git refuses loudly and ``lc init`` is the + remedy. The filter drivers and the four hooks are written in stock + form (PATH-resolved, as ``git annex init`` writes them) or pinned to + one absolute executable — the caller decides which, because only it + knows what the researcher's shell resolves. Hooks keep git-annex's + own shebang and marker comment, so git-annex still recognises them + as machinery; a hook without the marker is the user's and is left + alone. + + Args: + directory: A directory inside the repository. + pin: The absolute git-annex to pin everything to, or ``None`` + for the stock, PATH-resolved form. + """ + _git(["config", "filter.annex.required", "true"], cwd=directory) + config, hook = _annex_spellings(pin) + for key, command in _FILTERS.items(): + _git(["config", key, command.format(annex=config)], cwd=directory) + # A hooks directory that is not there holds nothing git would run, so + # there is nothing to converge in it — `git init` always creates one. + hooks = _hooks_dir(directory) + if not hooks.is_dir(): + return + marker = _HOOK_MARKER + (", pinned by lightcone-cli" if pin else "") + for name, command in _HOOKS.items(): + path = hooks / name + if path.exists() and _HOOK_MARKER not in path.read_text(): + continue + path.write_text(f"#!/bin/sh\n# {marker}\n{command.format(annex=hook)}\n") + path.chmod(0o755) + + +def annex_plumbing_current(directory: Path, *, stock_ok: bool) -> bool: + """Whether the plumbing needs no write on this host. + + Current means: ``required=true`` is set, and the filter drivers and + every hook lc owns resolve git-annex the same one way — a stock, + PATH-resolved spelling where the shell can honour it, or a pin that + still runs. A pin that works stays current even when PATH could now + serve (both work, and rewriting would flip-flop between a tool + install and a uvx cache); only a broken state reports otherwise. + + Args: + directory: A directory inside the repository. + stock_ok: Whether PATH-resolved plumbing works for the + researcher's shell — the caller's :func:`~lightcone.engine. + project.ambient_annex` question. + + Returns: + True if a convergence would write nothing. + """ + if _config(directory, "filter.annex.required") != "true": + return False + pin = pinned_annex(directory) + if pin is None and not stock_ok: + return False + if pin is not None and not annex_runs(pin): + return False + config, hook = _annex_spellings(pin) + if any(_config(directory, k) != c.format(annex=config) for k, c in _FILTERS.items()): + return False + hooks = _hooks_dir(directory) + if not hooks.is_dir(): + return True # nothing git would run — the write path's judgment too + for name, command in _HOOKS.items(): + path = hooks / name + text = path.read_text() if path.exists() else "" + if text and _HOOK_MARKER not in text: + continue # the user's hook — theirs in the write path too + if command.format(annex=hook) not in text: + return False + return True + + +def _annex_spellings(pin: Path | None) -> tuple[str, str]: + """How the config and the hooks spell git-annex, stock or pinned. + + Stock differs between the two sites — the filters call the + executable (``git-annex``), the hooks go through git's dispatch + (``git annex``) — while a pin is one absolute path at both. + """ + if pin is None: + return "git-annex", "git annex" + quoted = shlex.quote(pin.as_posix()) + return quoted, quoted + + +def _config(directory: Path, key: str) -> str: + """Read one repository config value, empty when unset.""" + found = project._run(["git", "config", "--get", key], cwd=directory) + return str(found.stdout or "").strip() if found.returncode == 0 else "" + + +def _hooks_dir(directory: Path) -> Path: + """Where this repository's hooks live. + + Asked of git rather than spelled ``.git/hooks``: in a linked + worktree ``.git`` is a file, hooks live in the common directory, and + ``core.hooksPath`` can move them anywhere — all questions + ``--git-path`` already answers. + """ + hooks = _git(["rev-parse", "--git-path", "hooks"], cwd=directory).strip() + return (directory / hooks).resolve() + + # ============================================================================= # What a run does to the repository # ============================================================================= diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 836ca69..6950e10 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -7,6 +7,7 @@ import re import shutil import subprocess +import sys import uuid from collections.abc import Callable, Sequence from dataclasses import asdict, dataclass, field @@ -312,6 +313,62 @@ def require_git_annex() -> None: ) +def bundled_annex() -> Path | None: + """Locate the git-annex this engine carries, beside its interpreter. + + Every install of lightcone-cli links the git-annex wheel's entry + points into the same bin directory as ``lc`` and its python, so the + sibling of :data:`sys.executable` is the one place the engine's own + git-annex is by construction — never ambient ``PATH``, which is the + researcher's and answers a different question. + + Returns: + The executable, or ``None`` on a broken install. + """ + found = shutil.which("git-annex", path=str(_engine_bin())) + return Path(found) if found else None + + +def _engine_bin() -> Path: + """The bin directory the engine's executables live in. + + The *parent* of ``sys.executable``, resolved as a directory only: in + a venv the python file itself is a symlink to the managed + interpreter, and following it first would answer with a directory + that holds no entry point of ours. + """ + return Path(sys.executable).parent.resolve() + + +def ambient_annex() -> Path | None: + """Locate the git-annex the researcher's *own* shell would find. + + ``PATH`` minus the engine's bin directory: launchers front that + directory on lc's environment (uvx, ``uv run``), so a naive + ``which`` from inside lc always finds the bundled copy — while the + shell the researcher types ``git add`` into may hold nothing of the + sort. This is the probe that decides whether stock, PATH-resolved + annex plumbing actually works for them. + + Returns: + The executable as the shell would resolve it, or ``None``. + """ + ours = _engine_bin() + kept = [ + entry + for entry in os.environ.get("PATH", "").split(os.pathsep) + if entry and Path(entry).resolve() != ours + ] + found = shutil.which("git-annex", path=os.pathsep.join(kept)) + # Directories are compared, never the file's own symlink target: a + # tool install's `~/.local/bin/git-annex` points into the engine's + # venv, and it is still genuinely the shell's — PATH keeps resolving + # it for exactly as long as that install exists. + if found is None or Path(found).parent.resolve() == ours: + return None + return Path(found) + + def uv_prefix(directory: Path, *, sync: bool) -> list[str]: """Build the ``uv run`` hop that pins a command to a project. @@ -407,11 +464,12 @@ def _converge_dataset(c: _Converger, directory: Path) -> None: """ c.item(".git", _in_repository(directory), lambda: dataset.init_git(directory)) # After the item above, so a fresh project has a repository to annex. - c.item( - "git-annex", - _can_ask_git(directory) and dataset.is_annexed(directory), - lambda: dataset.init_annex(directory), - ) + # `annexed` is read *before* the item applies: in write mode the init + # runs inline, and the plumbing item below must still know whether + # this run created the annex (created) or found one (repaired). + annexed = _can_ask_git(directory) and dataset.is_annexed(directory) + c.item("git-annex", annexed, lambda: dataset.init_annex(directory)) + _converge_annex_plumbing(c, directory, present=annexed) attributes = directory / ".gitattributes" # Read before the repair, not after: the check is on the text a repair # would leave behind, and check mode never writes one. @@ -440,6 +498,60 @@ def _converge_dataset(c: _Converger, directory: Path) -> None: _converge_committable(c, directory) +def _converge_annex_plumbing(c: _Converger, directory: Path, *, present: bool) -> None: + """The plumbing that lets the researcher's *own* git find git-annex. + + ``git annex init`` wires the filter drivers and four hooks to a bare + ``git-annex``, resolved from the PATH of whichever git runs — and the + researcher's shell, unlike lc's environment, may hold no git-annex at + all (the engine invoked through uvx puts nothing on it). Measured: + a ``git add`` whose clean filter cannot start prints an error, exits + 0, and stages the raw bytes into git history. So convergence owns the + plumbing: ``filter.annex.required=true`` always, making that failure + git's own hard refusal with ``lc init`` as the remedy — and where the + shell would not resolve git-annex, the filter and hooks are pinned to + the engine's bundled executable by absolute path, so the ordinary + ``git add`` the docs promise keeps working. Where the shell *does* + resolve one, the stock form is kept: it already works, and it outlives + a prunable engine path. + + The repair rule is asymmetric on purpose. A pin that still runs is + left alone even when it is not this engine's copy or PATH could now + serve — both work, and rewriting would flip-flop between a tool + install and a uvx cache. Only a broken state is rewritten: toward + stock when the shell resolves git-annex, else to this engine's copy. + + Args: + c: The converger. + directory: The project root. + present: Whether the annex existed before this convergence — + plumbing written over a fresh ``git annex init`` is created, + plumbing brought up to date on an adopted annex is repaired. + """ + recorded = dataset.pinned_annex(directory) if present else None + keep = recorded if recorded is not None and dataset.annex_runs(recorded) else None + ambient = ambient_annex() + target = keep or (None if ambient is not None else bundled_annex()) + if keep is None and ambient is None and target is None: + # Near-unreachable — require_git_annex found one somewhere — but a + # missing bundled copy must degrade to advice, not a wrong write. + c.warn( + "the annex plumbing was left as git-annex wrote it: no git-annex " + "on PATH for the shell's own `git add` to find, and lc's bundled " + "copy is missing beside its interpreter. " + "`uv tool install --force lightcone-cli` repairs the install." + ) + return + c.item( + "annex-plumbing", + present, + lambda: dataset.converge_annex_plumbing(directory, target), + is_current=lambda: dataset.annex_plumbing_current( + directory, stock_ok=ambient is not None + ), + ) + + def _converge_committable(c: _Converger, directory: Path) -> None: """Refuse to call a project converged while its outputs are unignorable. diff --git a/tests/conftest.py b/tests/conftest.py index 51536f3..0e72a5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -102,6 +102,10 @@ def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: # rather than on disk because `.git` is a *file* in a linked # worktree, so there is nowhere inside it to leave a marker. annexed: set[Path] = set() + # What `git config` has been told, per repository — the annex + # plumbing converges through config reads and writes, and a fake + # that forgot them would report the same items repaired forever. + config: dict[tuple[Path, str], str] = {} def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: calls.append(list(argv)) @@ -115,11 +119,23 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: elif argv[:2] == ["uv", "sync"]: (project / ".venv" / "bin").mkdir(parents=True, exist_ok=True) elif argv[:2] == ["git", "init"]: - (cwd / ".git").mkdir(exist_ok=True) + # With the hooks directory, as the real `git init` makes it — + # the annex plumbing writes hook files into it. + (cwd / ".git" / "hooks").mkdir(parents=True, exist_ok=True) elif argv[:3] == ["git", "annex", "init"]: annexed.add(_repo(cwd)) elif argv[:2] == ["git", "config"] and argv[-1] == "annex.uuid": return MagicMock(returncode=0 if _repo(cwd) in annexed else 1) + elif argv[:3] == ["git", "config", "--get"]: + value = config.get((_repo(cwd), argv[3])) + code = 0 if value is not None else 1 + return MagicMock(returncode=code, stdout=(value or "") + "\n", stderr="") + elif argv[:2] == ["git", "config"] and len(argv) == 4 and not argv[2].startswith("-"): + config[(_repo(cwd), argv[2])] = argv[3] + elif argv[:3] == ["git", "rev-parse", "--git-path"]: + return MagicMock( + returncode=0, stdout=str(_repo(cwd) / ".git" / argv[3]) + "\n", stderr="" + ) elif argv[:2] == ["git", "check-ignore"]: return _fake_check_ignore(cwd, argv[-1]) return MagicMock(returncode=0, stdout="", stderr="") diff --git a/tests/test_dataset.py b/tests/test_dataset.py index c21e649..591bc47 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -10,6 +10,7 @@ from __future__ import annotations import shutil +import subprocess from pathlib import Path import pytest @@ -499,6 +500,147 @@ def test_the_worker_and_the_shim_are_never_console_scripts() -> None: assert entry.value.startswith("lightcone.cli") or entry.value in theirs, entry +# ---- the annex plumbing, against a real shell ------------------------------- + + +def _researcher_shell(tmp_path: Path) -> dict[str, str] | None: + """The environment a researcher's own `git add` runs in when nothing + put git-annex on their PATH — the uvx case. The system directories + carry git and the usual utilities; whether they also carry a + git-annex is the host's business, and where they do there is nothing + for these tests to prove. Probed by listing the directories rather + than `shutil.which`, which the autouse `tools` fixture narrows for + exactly the three tools this asks about.""" + dirs = [Path("/usr/bin"), Path("/bin")] + if not any((d / "git").is_file() for d in dirs): + return None + if any((d / "git-annex").exists() for d in dirs): + return None + return {"PATH": ":".join(str(d) for d in dirs), "HOME": str(tmp_path)} + + +def _shell_git(repo: Path, env: dict[str, str], *argv: str) -> subprocess.CompletedProcess[str]: + """Run git exactly as that shell would — not through `project._run`, + whose environment is lc's own and always resolves the bundled annex.""" + git = next(p for d in env["PATH"].split(":") if (p := Path(d) / "git").is_file()) + return subprocess.run( + [str(git), *argv], cwd=repo, env=env, capture_output=True, text=True, check=False + ) + + +@pytest.fixture +def no_annex_shell(tmp_path: Path) -> dict[str, str]: + env = _researcher_shell(tmp_path) + if env is None: + pytest.skip("the system PATH itself resolves git-annex (or lacks git)") + return env + + +def test_pinned_plumbing_annexes_content_with_no_git_annex_on_path( + repo: Path, no_annex_shell: dict[str, str] +) -> None: + """The pin's whole claim, measured: with the filter and hooks pinned + to the engine's git-annex by absolute path, the ordinary `git add` + the docs promise stages an annex pointer — on a shell that could not + resolve git-annex at all.""" + annex = project.bundled_annex() + assert annex is not None, "the test environment carries the git-annex wheel" + dataset.converge_annex_plumbing(repo, annex) + (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) + + added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") + assert added.returncode == 0, added.stderr + staged = _shell_git(repo, no_annex_shell, "cat-file", "-p", ":data/catalog.fits") + assert staged.stdout.startswith("/annex/objects/"), "a pointer, not the bytes" + + # The commit exercises the pinned pre-commit hook the same way. + committed = _shell_git(repo, no_annex_shell, "commit", "-m", "input data") + assert committed.returncode == 0, committed.stderr + assert _annexed(repo, repo / "data" / "catalog.fits") + + +def test_required_true_makes_the_missing_filter_loud_not_silent( + repo: Path, no_annex_shell: dict[str, str] +) -> None: + """The safety net on its own: stock plumbing plus `required=true`, + and a shell without git-annex gets git's own hard refusal — nothing + staged, exit nonzero — instead of raw bytes in git history. `lc init` + is the remedy; it pins the plumbing on exactly such a host.""" + dataset._git(["config", "filter.annex.required", "true"], cwd=repo) + (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) + + added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") + + assert added.returncode != 0 + assert "clean filter 'annex' failed" in added.stderr + staged = _shell_git(repo, no_annex_shell, "diff", "--cached", "--name-only") + assert staged.stdout.strip() == "" + + +def test_stock_plumbing_without_required_stages_raw_bytes_silently( + repo: Path, no_annex_shell: dict[str, str] +) -> None: + """The hazard the plumbing convergence exists for, recorded as + measured: as `git annex init` leaves a repository, a shell without + git-annex prints an error, *exits 0*, and stages the raw bytes into + git history — a 2 GB dataset in git proper, on every clone, forever. + If this ever starts failing, git changed the behavior and the + `required=true` net is worth re-examining.""" + (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) + + added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") + + assert added.returncode == 0 + assert "git-annex: command not found" in added.stderr + staged = _shell_git(repo, no_annex_shell, "cat-file", "-p", ":data/catalog.fits") + assert not staged.stdout.startswith("/annex/objects/") + + +def test_stock_spellings_mirror_what_git_annex_itself_writes(repo: Path) -> None: + """Mirrored, never invented: with only the `required` flag added to + what `git annex init` wrote, the plumbing is already current — so an + upstream change to the stock filter or hook spelling fails here, + rather than reporting every existing project repaired forever.""" + dataset._git(["config", "filter.annex.required", "true"], cwd=repo) + assert dataset.annex_plumbing_current(repo, stock_ok=True) + # And stock is not current where the shell cannot resolve it. + assert not dataset.annex_plumbing_current(repo, stock_ok=False) + + +def test_the_pin_is_recorded_and_its_health_probed(repo: Path, tmp_path: Path) -> None: + """The repair rule works from the recorded path: readable back out of + the filter config, current while it runs, drift the moment it dies.""" + annex = tmp_path / "engine" / "git-annex" + annex.parent.mkdir() + annex.write_text("#!/bin/sh\nexit 0\n") + annex.chmod(0o755) + + dataset.converge_annex_plumbing(repo, annex) + + assert dataset.pinned_annex(repo) == annex + assert dataset.annex_plumbing_current(repo, stock_ok=False) + annex.unlink() + assert not dataset.annex_plumbing_current(repo, stock_ok=True) + + +def test_repair_to_stock_rewrites_a_pinned_repository(repo: Path, tmp_path: Path) -> None: + """The way back: a broken pin on a host whose PATH resolves git-annex + is returned to the stock form, byte-for-byte what `git annex init` + writes plus the required flag — and git-annex still recognises the + hooks as its own.""" + annex = tmp_path / "engine" / "git-annex" + annex.parent.mkdir() + annex.write_text("#!/bin/sh\nexit 0\n") + annex.chmod(0o755) + dataset.converge_annex_plumbing(repo, annex) + + dataset.converge_annex_plumbing(repo, None) + + assert dataset.pinned_annex(repo) is None + assert dataset.annex_plumbing_current(repo, stock_ok=True) + hook = (repo / ".git" / "hooks" / "pre-commit").read_text() + assert "git annex pre-commit ." in hook + assert str(annex) not in hook # ---- who last wrote a path ------------------------------------------------- diff --git a/tests/test_project.py b/tests/test_project.py index 463af2e..02bc68a 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -212,10 +212,12 @@ def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: tracks has to be something git can carry, or a fresh clone reports drift forever. - Two items are exempt, and both for the same reason — they are local + Three items are exempt, and all for the same reason — they are local state git does not clone. `.venv` is git-ignored and rebuilt from the - lock, and `git clone` of an annexed repository leaves the annex - uninitialized until someone runs `git annex init`.""" + lock, `git clone` of an annexed repository leaves the annex + uninitialized until someone runs `git annex init`, and the annex + plumbing lives in `.git` (config and hooks), which a clone starts + fresh.""" project = tmp_path / "proj" converge(project) @@ -230,7 +232,7 @@ def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: (clone / ".git").mkdir() report = converge(clone, write=False) - assert report.created == ["git-annex", ".venv"], report.created + assert report.created == ["git-annex", "annex-plumbing", ".venv"], report.created def test_converge_repairs_a_missing_piece(tmp_path: Path) -> None: @@ -505,6 +507,243 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: converge(tmp_path / "proj") +# ---- the annex plumbing ---------------------------------------------------- +# +# `git annex init` wires the filter and hooks to a bare `git-annex`, +# resolved from the PATH of whichever git runs — and the researcher's +# shell may hold none (the engine invoked through uvx puts nothing on +# it), where a plain `git add` exits 0 and stages raw bytes into git +# history. Convergence owns the plumbing: `required=true` always, and a +# pin to the engine's own executable exactly where PATH would not serve. + + +def _engine_annex(tmp_path: Path, name: str = "engine") -> Path: + """A stand-in for the bundled git-annex: a real executable file, so + the repair rule's exists-and-runs probe has something to answer about.""" + bindir = tmp_path / name + bindir.mkdir(exist_ok=True) + annex = bindir / "git-annex" + annex.write_text("#!/bin/sh\nexit 0\n") + annex.chmod(0o755) + return annex + + +def _annex_resolution( + monkeypatch: pytest.MonkeyPatch, *, ambient: Path | None, bundled: Path | None +) -> None: + """Pin what the two PATH questions answer, host-independently.""" + from lightcone.engine import project as project_mod + + monkeypatch.setattr(project_mod, "ambient_annex", lambda: ambient) + monkeypatch.setattr(project_mod, "bundled_annex", lambda: bundled) + + +def _config_writes(tools: list[list[str]]) -> dict[str, str]: + """The ``git config `` writes, last value per key.""" + return { + c[2]: c[3] + for c in tools + if c[:2] == ["git", "config"] and len(c) == 4 and not c[2].startswith("-") + } + + +def test_ambient_annex_never_answers_with_the_engines_own_copy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe asks what the *researcher's* shell resolves, and lc's own + environment always fronts the engine's bin (uv run, uvx) — so that + directory must be blind-spotted even when it is all PATH holds. In a + venv `sys.executable` is a symlink: resolving it as a file first made + the engine's directory stop matching itself, and the probe reported + the bundled copy as the shell's.""" + import sys + + from lightcone.engine import project as project_mod + + monkeypatch.setenv("PATH", str(Path(sys.executable).parent)) + assert project_mod.ambient_annex() is None + + +def test_bundled_annex_is_the_engines_sibling() -> None: + """The suite runs where the git-annex wheel is installed, so the + engine's own copy resolves beside its interpreter — the path every + pin is made of.""" + from lightcone.engine import project as project_mod + + assert project_mod.bundled_annex() == project_mod._engine_bin() / "git-annex" + + +def test_a_shell_without_git_annex_gets_the_plumbing_pinned( + tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """The uvx case: nothing lands on the researcher's PATH, so their own + `git add` must reach the engine's git-annex by absolute path — filter + drivers and all four hooks, the post-receive guard included.""" + annex = _engine_annex(tmp_path) + _annex_resolution(monkeypatch, ambient=None, bundled=annex) + project = tmp_path / "proj" + + report = converge(project) + + assert "annex-plumbing" in report.created + written = _config_writes(tools) + assert written["filter.annex.required"] == "true" + assert written["filter.annex.process"] == f"{annex} filter-process" + assert written["filter.annex.smudge"] == f"{annex} smudge -- %f" + assert written["filter.annex.clean"] == f"{annex} smudge --clean -- %f" + hooks = project / ".git" / "hooks" + assert f"{annex} pre-commit ." in (hooks / "pre-commit").read_text() + assert f"{annex} smudge --update" in (hooks / "post-checkout").read_text() + assert f"{annex} smudge --update" in (hooks / "post-merge").read_text() + receive = (hooks / "post-receive").read_text() + assert f"if {annex} post-receive --help" in receive + assert "git annex" not in receive + for name in ("pre-commit", "post-checkout", "post-merge", "post-receive"): + text = (hooks / name).read_text() + assert text.startswith("#!/bin/sh\n") + assert "automatically configured by git-annex, pinned by lightcone-cli" in text + + +def test_a_shell_with_git_annex_keeps_stock_plumbing_plus_the_required_net( + tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """Where the researcher's PATH already resolves git-annex, the stock + form works and outlives any engine path — only `required=true` is + added, so a PATH that later loses git-annex fails loudly instead of + staging raw bytes.""" + _annex_resolution( + monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path) + ) + project = tmp_path / "proj" + + report = converge(project) + + assert "annex-plumbing" in report.created + written = _config_writes(tools) + assert written["filter.annex.required"] == "true" + assert written["filter.annex.process"] == "git-annex filter-process" + hook = (project / ".git" / "hooks" / "pre-commit").read_text() + assert "git annex pre-commit ." in hook + + +@pytest.mark.parametrize("ambient", [None, Path("/usr/bin/git-annex")]) +def test_converged_plumbing_reconverges_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ambient: Path | None +) -> None: + project = tmp_path / "proj" + _annex_resolution(monkeypatch, ambient=ambient, bundled=_engine_annex(tmp_path)) + converge(project) + + report = converge(project) + + assert report.converged + assert "annex-plumbing" in report.unchanged + + +def test_a_dead_pin_is_repaired_to_the_current_engine( + tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """A uv cache prune or a moved engine breaks the recorded path; + `required=true` makes that loud, and the next convergence points the + plumbing at the engine that is actually here.""" + project = tmp_path / "proj" + old = _engine_annex(tmp_path, "old-engine") + _annex_resolution(monkeypatch, ambient=None, bundled=old) + converge(project) + old.unlink() + new = _engine_annex(tmp_path, "new-engine") + _annex_resolution(monkeypatch, ambient=None, bundled=new) + + report = converge(project) + + assert "annex-plumbing" in report.repaired + assert _config_writes(tools)["filter.annex.process"] == f"{new} filter-process" + assert f"{new} pre-commit ." in (project / ".git" / "hooks" / "pre-commit").read_text() + + +def test_a_dead_pin_returns_to_stock_when_the_shell_has_git_annex( + tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """Repair goes toward the durable answer: a PATH-resolved git-annex + outlives any engine install, so a broken pin on a host that has one + is put back the way `git annex init` writes it.""" + project = tmp_path / "proj" + old = _engine_annex(tmp_path, "old-engine") + _annex_resolution(monkeypatch, ambient=None, bundled=old) + converge(project) + old.unlink() + _annex_resolution( + monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path) + ) + + report = converge(project) + + assert "annex-plumbing" in report.repaired + assert _config_writes(tools)["filter.annex.process"] == "git-annex filter-process" + assert "git annex pre-commit ." in (project / ".git" / "hooks" / "pre-commit").read_text() + + +def test_a_working_pin_survives_git_annex_appearing_on_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Both forms work, so rewriting would only flip-flop between a tool + install and a uvx cache — a healthy state is never rewritten.""" + project = tmp_path / "proj" + pinned = _engine_annex(tmp_path) + _annex_resolution(monkeypatch, ambient=None, bundled=pinned) + converge(project) + _annex_resolution( + monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path, "other") + ) + + report = converge(project) + + assert report.converged + assert "annex-plumbing" in report.unchanged + assert f"{pinned} pre-commit ." in (project / ".git" / "hooks" / "pre-commit").read_text() + + +def test_check_mode_reports_plumbing_drift_and_writes_nothing( + tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """The PATH that satisfied stock plumbing at init can stop resolving + git-annex later; `--check` must say so — and, as everywhere, without + fixing it.""" + project = tmp_path / "proj" + _annex_resolution( + monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path) + ) + converge(project) + _annex_resolution(monkeypatch, ambient=None, bundled=_engine_annex(tmp_path)) + hooks_before = (project / ".git" / "hooks" / "pre-commit").read_bytes() + tools.clear() + + report = converge(project, write=False) + + assert "annex-plumbing" in report.repaired + assert not report.converged + assert _config_writes(tools) == {} + assert (project / ".git" / "hooks" / "pre-commit").read_bytes() == hooks_before + + +def test_a_users_own_hook_is_never_touched( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The marker `git annex init` leaves is the ownership test — a hook + without it is the user's, skipped by the write and by the drift + check alike, exactly as git-annex itself treats it.""" + project = tmp_path / "proj" + _annex_resolution(monkeypatch, ambient=None, bundled=_engine_annex(tmp_path)) + converge(project) + theirs = "#!/bin/sh\nblack --check .\n" + (project / ".git" / "hooks" / "pre-commit").write_text(theirs) + + report = converge(project) + + assert report.converged + assert (project / ".git" / "hooks" / "pre-commit").read_text() == theirs + + # ---- refusals ------------------------------------------------------------- From 2351d66ce8094538cb9e6ff3542e7f9004688b7d Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 16:28:40 +0200 Subject: [PATCH 2/5] Reduce the annex plumbing to the one key that prevents corruption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin is reverted before merge; `filter.annex.required=true` stays. `git annex init` resolves git-annex from the PATH of whichever git runs, so a researcher whose shell has none gets a `git add` that prints an error, exits 0, and stages the raw bytes into git history. The flag makes that git's own hard refusal instead, and it is the whole fix: one per-repository config key, no absolute path, nothing that can go stale. Why the pin goes: - It wrote durable repository state pointing at a prunable uvx cache. - `core.hooksPath` can resolve outside the repository, so the four hooks could land in a machine-shared directory and break commits in unrelated repositories. - uv documents uvx environments as disposable; `uv cache prune` deletes them, and the path is a symlink into cache internals. - git-lfs solves the same problem with the same mechanism and has never baked a path: `filter.lfs.required = true` beside PATH-relative drivers. Reachability is the install's job, not the repository's: `uv tool install lightcone-cli` links the git-annex wheel's entry points beside `lc`, so PATH resolves it for free. uvx runs lc fine but cannot support a bare `git add`, which the docs now say. Deleted with the pin: _FILTERS, _HOOKS, _HOOK_MARKER, pinned_annex, annex_runs, _annex_spellings, _hooks_dir, _config, and project's bundled_annex / ambient_annex / _engine_bin. No hook file is written and no filter driver git-annex wrote is rewritten. Both measured tests stay: the loud refusal, and the silent raw-bytes staging it prevents. The stderr assertion no longer pins bash's wording, which dash — /bin/sh on Debian and Ubuntu — does not produce. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016MWNSzVFUMmieXA9ayPtbL --- CLAUDE.md | 62 +++++---- docs/api/dataset.md | 12 +- docs/api/project.md | 14 +- docs/architecture.md | 12 +- docs/cli/init.md | 27 ++-- docs/user/troubleshooting.md | 21 +-- src/lightcone/engine/dataset.py | 198 +++----------------------- src/lightcone/engine/project.py | 124 ++--------------- tests/conftest.py | 14 +- tests/test_dataset.py | 94 ++----------- tests/test_project.py | 237 ++++++-------------------------- 11 files changed, 164 insertions(+), 651 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f8d1a32..66cb791 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -327,7 +327,7 @@ user owns: | `.python-version` | The exact patch of the interpreter `lc` is running on | | `uv.lock`, `.venv` | **Derived** — converged by correctness, not existence: `uv lock --check` / `uv sync --locked --exact --check` decide, then `uv lock` / `uv sync --locked --exact --compile-bytecode` repair | | `.gitignore` | One managed block of patterns; convergence ensures each is present | -| `.git` + the annex | `git init` then `git annex init` — results are versioned in the project's own repository — then the **annex plumbing** (`filter.annex.required=true` always; filter + hooks pinned to the bundled git-annex only where the shell's `PATH` resolves none — see the storage invariants) | +| `.git` + the annex | `git init` then `git annex init` — results are versioned in the project's own repository — then **`filter.annex.required=true`**, the one key git-annex does not set and the difference between a loud refusal and silent corruption (see the storage invariants) | | `.gitattributes` | The storage policy: what git-annex holds and what git carries. Line-managed, like `.gitignore` | | `.datalad/config` | A `datalad.dataset.id` UUID, generated once. Read back only by `dataset.dataset_id`, through `git config -f`, for the run record's `dsid` | | `data/` + `README.md` | Where declared inputs live; annexed, and committed before anything computes on them | @@ -342,8 +342,9 @@ user owns: Universes are discovered by `glob("*.yaml")`, which is empty-not-error on a missing directory. `tests/test_project.py::test_a_clone_of_a_converged_project_is_converged` pins this: a clone must need nothing but `.venv` and `git annex init`. - Those (with the annex plumbing) are the exemptions, and for one - reason — they are local state git does not clone. + Those (with the annex filter, a `.git/config` entry) are the + exemptions, and for one reason — they are local state git does not + clone. - **Convergence, not scaffolding.** Each item is created if missing, offered to a conservative `repair(text) -> str | None` hook otherwise, and left alone when the hook returns `None`. `--check` computes the @@ -547,34 +548,35 @@ from which `git` itself runs. Nothing records the annex version anyway **"By construction" covers every install channel — except the researcher's own shell**, which is the one place `filter=annex` sends -git looking for git-annex (the filter config and the four hooks -`git annex init` writes, all `PATH`-resolved). Under `uvx` nothing -lands on the user's `PATH`, and a `git add` whose clean filter cannot -start prints an error, **exits 0, and stages the raw bytes into git -history** (measured, and pinned by +git looking for git-annex (the filter config `git annex init` writes is +`PATH`-resolved). Under `uvx` nothing lands on the user's `PATH`, and a +`git add` whose clean filter cannot start prints an error, **exits 0, +and stages the raw bytes into git history** (measured, and pinned by `test_stock_plumbing_without_required_stages_raw_bytes_silently`). So -dataset convergence owns the **annex plumbing** as an item -(`annex-plumbing`, 2026-08): `filter.annex.required=true` always — -git's own hard failure instead of silent corruption, with `lc init` as -the remedy — and the filter drivers plus hooks pinned to the absolute -path of the engine's bundled git-annex (`project.bundled_annex`, the -sibling of `sys.executable`) **only where the shell's own `PATH` -resolves no git-annex** (`project.ambient_annex`, probed with the -engine's bin stripped out). Where `PATH` serves, stock plumbing stays: -it already works and outlives any prunable engine path. Repair is -asymmetric by decision: a pin that runs is never rewritten, even when -it is not this engine's copy or ambient git-annex has appeared (both -work; rewriting flip-flops between a tool install and a uvx cache) — -only a broken state is rewritten, toward stock when the shell resolves -git-annex, else toward this engine's copy. Hooks keep git-annex's own -marker comment ("automatically configured by git-annex"), which is -also the ownership test — a hook without it is the user's and is never -touched. The stock spellings are mirrored, never invented: -`test_stock_spellings_mirror_what_git_annex_itself_writes` pins them -against a real `git annex init`. `require_git_annex` stays a `PATH` -check, deliberately — it gates lc's *own* `git annex` subprocesses, -which still dispatch from lc's environment, where every install -channel fronts the bundled copy. +dataset convergence sets one key as an item (`annex-filter`, 2026-08): +**`filter.annex.required=true`, always** — git's own hard failure +instead of silent corruption. + +**That flag is the whole of it, by decision.** lc writes no absolute +path into a repository and rewrites no filter driver or hook: how git +dispatches git-annex stays `PATH` resolution, which is git-annex's own +design and git-lfs's too (it writes `filter.lfs.required = true` beside +`PATH`-relative drivers and has never baked a path). Pinning the filter +and the four hooks to the engine's bundled executable was implemented +and reverted before merge: it writes durable state pointing at a +prunable `uvx` cache, `core.hooksPath` can send the hooks into a +machine-shared directory that breaks commits in unrelated +repositories, and uv documents `uvx` environments as disposable. The +reachability problem is the *install's* to solve, not the +repository's — `uv tool install lightcone-cli` links the git-annex +wheel's entry points beside `lc` (verified), so `ambient` git-annex is +on `PATH` for free and the whole question disappears. `uvx` is fine for +running lc and cannot support the researcher's bare `git add`; that is +what the docs and the troubleshooting entry say, and reporting it from +`lc init`/`lc status` is an open follow-up, not a promise made here. +`require_git_annex` stays a `PATH` check, deliberately — it gates lc's +*own* `git annex` subprocesses, which dispatch from lc's environment, +where every install channel fronts the bundled copy. **A project can sit inside a larger repository, so `dataset.status` is scoped and relativised.** `lc init subdir/` adopts an enclosing work diff --git a/docs/api/dataset.md b/docs/api/dataset.md index 50c07e0..a6362ac 100644 --- a/docs/api/dataset.md +++ b/docs/api/dataset.md @@ -20,9 +20,8 @@ Source: `src/lightcone/engine/dataset.py` (+ | `last_writer(root, dir)` | Who last touched an output's directory — the foreign-write question. Answers "cannot say" as empty, never an error. | | `require_committer(root)` | Refuses a repository with no git identity, before any recipe spends time. Asked as `git var`, the question a commit itself asks. | | `dataset_id(root)` | The DataLad dataset UUID, read via `git config -f`. | -| `converge_annex_plumbing(root, pin)` | Write the plumbing the researcher's own `git add` dispatches git-annex through: `required=true` always, filter drivers + four hooks in stock (`PATH`-resolved) or pinned (absolute-path) form. | -| `annex_plumbing_current(root, *, stock_ok)` | The no-write drift probe for the same, `lc init --check`'s question. | -| `pinned_annex(root)` / `annex_runs(path)` | The recorded pin, and whether it still runs — the repair rule's two reads. | +| `require_annex_filter(root)` | Set `filter.annex.required=true`, so a `git add` that cannot reach git-annex fails loudly instead of staging raw bytes. | +| `annex_filter_required(root)` | Whether that flag is already set — `lc init --check`'s question. | ## What must stay true @@ -57,9 +56,10 @@ Source: `src/lightcone/engine/dataset.py` (+ an error, exits 0, and stages the raw bytes into git history — measured, and pinned by `test_stock_plumbing_without_required_stages_raw_bytes_silently`. - The stock filter/hook spellings are mirrored from what - `git annex init` itself writes, never invented, and a hook without - git-annex's own marker comment is the user's — never rewritten. + It is the *only* thing convergence adds to what `git annex init` + wrote: no filter driver is rewritten, and no hook is touched, so how + git dispatches git-annex stays git-annex's own business and stays + resolved from `PATH`. ## Tests diff --git a/docs/api/project.md b/docs/api/project.md index 2684576..d2d0797 100644 --- a/docs/api/project.md +++ b/docs/api/project.md @@ -17,7 +17,6 @@ Source: `src/lightcone/engine/project.py` (+ | `declared_project()` | The weaker question — what the repository carries, without `.venv`. One caller: the worker entry point, which builds the venv a moment later. | | `mode(root)` | `"direct"` or `"containerized"` — presence of `[tool.lightcone.image]`, nothing else. | | `uv_prefix(root, *, sync)` | The one spelling of the project uv hop. Callers differ only in `sync`: a probe converges the environment, a recipe must not. | -| `bundled_annex()` / `ambient_annex()` | The two git-annex questions: the engine's own copy (beside `sys.executable`) and the one the researcher's *shell* would resolve (`PATH` minus the engine's bin). The annex-plumbing item decides between stock and pinned form from the second, and pins to the first. | | `project_name(dir)` | PEP 503-ish name from the directory name. | | `_run` / `_check_call` | Every external tool invocation, and the suite's one monkeypatch point. | | `ProjectError` | The engine's one exception; the CLI translates it once. | @@ -40,15 +39,14 @@ Source: `src/lightcone/engine/project.py` (+ template). - **Only what git can carry is converged.** No `src/`, no empty directories — a clone must need nothing but `.venv`, `git annex - init` and the annex plumbing (all three local state git does not + init` and the annex filter (all three local state git does not clone), and `test_a_clone_of_a_converged_project_is_converged` pins it. -- **The annex plumbing repairs only broken states.** - `filter.annex.required=true` always; the filter and hooks pinned to - the engine's git-annex only where the researcher's shell resolves - none; a working pin never rewritten (both forms work — rewriting - flip-flops between installs), a dead one rewritten toward stock when - the shell has git-annex, else toward this engine's copy. +- **The annex filter is one config key.** `filter.annex.required=true`, + always, so a `git add` that cannot reach git-annex refuses instead of + silently staging raw bytes. Nothing else about how git dispatches + git-annex is lc's to write: the filter drivers and hooks stay exactly + as `git annex init` left them, resolved from `PATH`. - **There is no discovery.** The invoked directory is the project or it is a clean error; every uv call carries an explicit `--project`. - **Templates are files** (`templates/files/*.tmpl`, `string.Template` diff --git a/docs/architecture.md b/docs/architecture.md index 85da6c4..f1610ab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -105,12 +105,12 @@ default; `data/` and `results/` opt out). A researcher only ever types ordinary `git add` / `git commit`. That ordinary `git add` dispatches git-annex from the *researcher's* -`PATH`, and a shell that cannot resolve it silently stages raw bytes -into git history — so `lc init` also owns the **annex plumbing**: -`filter.annex.required=true` always (the same situation becomes git's -own hard failure, remedied by `lc init`), and the filter plus hooks -pinned to the engine's bundled git-annex by absolute path exactly -where the shell's `PATH` resolves none. +`PATH`, and a shell that cannot resolve it stages the raw bytes into +git history while exiting 0 — so `lc init` sets +`filter.annex.required=true`, which makes git refuse loudly instead. +Getting git-annex onto that `PATH` is the install's job, not the +repository's: `uv tool install lightcone-cli` puts it there alongside +`lc`. Each output is committed with a **run record** — a `[DATALAD RUNCMD]` commit message whose `cmd` reconstructs the engine diff --git a/docs/cli/init.md b/docs/cli/init.md index 39103ce..26aa672 100644 --- a/docs/cli/init.md +++ b/docs/cli/init.md @@ -25,8 +25,9 @@ manages, and never overwrites files you own: - **Repaired** — derived artifacts that have drifted: a `uv.lock` that no longer matches `pyproject.toml`, a `.venv` that no longer matches the lock, a managed `.gitignore` or `.gitattributes` entry that a - newer `lc` added, annex plumbing that no longer resolves a working - git-annex. Repairs only ever append or rebuild derived state; + newer `lc` added, an annexed repository still missing the + `filter.annex.required` flag. Repairs only ever append or rebuild + derived state; hand-written lines are never reordered or removed. - **Blocked** — something convergence can see but must not fix by appending: a `.gitignore` rule that would silently swallow @@ -81,26 +82,20 @@ Two things it deliberately does *not* create: a `src/` directory directories), and any dependency in `pyproject.toml` — the lock carries only what *your* analysis imports, added with `uv add`. -Inside `.git`, convergence also owns the **annex plumbing** — the -filter configuration and hooks that let *your own* `git add` reach -git-annex (reported as the `annex-plumbing` item). Two guarantees: +Inside `.git`, convergence sets one configuration key — reported as the +`annex-filter` item: - `filter.annex.required=true`, always. Without it, a `git add` whose shell cannot find git-annex prints an error, **exits 0, and stages the raw bytes into git history** — a 2 GB dataset in git proper, on every clone, forever. With it, the same situation is a hard, loud failure and nothing is staged. -- Where your shell's `PATH` does not resolve `git-annex` (for example - when the engine was run through `uvx`, which puts nothing on your - `PATH`), the filter and hooks are pinned to the absolute path of the - git-annex that ships with `lc` itself, so the ordinary `git add` the - docs promise keeps working. Where your `PATH` does resolve one, the - stock, `PATH`-resolved form is kept — it already works, and it - outlives any particular `lc` install. - -A pinned path can die (a pruned `uv` cache, a moved install); thanks to -`required=true` that failure is loud, and the next `lc init` repairs -the pin. See + +That is the only thing `lc init` adds to what `git annex init` wrote. +How git finds git-annex is still ordinary `PATH` resolution, which is +why `lc` should be installed with `uv tool install lightcone-cli` — it +puts `git-annex` on your `PATH` alongside `lc`. If your `git add` ever +refuses, see [`fatal: … clean filter 'annex' failed`](../user/troubleshooting.md#fatal-clean-filter-annex-failed) in the troubleshooting guide. diff --git a/docs/user/troubleshooting.md b/docs/user/troubleshooting.md index d44c626..de6d10e 100644 --- a/docs/user/troubleshooting.md +++ b/docs/user/troubleshooting.md @@ -123,18 +123,23 @@ error: initialization for subprocess 'git-annex filter-process' failed fatal: data/catalog.fits: clean filter 'annex' failed ``` -Your `git add` could not reach git-annex — either your shell's `PATH` -lost it, or the project's plumbing is pinned to an `lc` install that no -longer exists (a pruned `uv` cache, a moved install). The remedy is -one command, from the project root: +Your shell's `PATH` has no `git-annex`, so git could not run the filter +that turns a large file into an annex pointer. **Nothing was staged**, +which is the point: without `filter.annex.required=true` — which +`lc init` sets — git would have exited 0 and committed the raw bytes +into history instead. + +`git-annex` ships with `lc`, so a tool install puts both on your `PATH`: ```bash -lc init +uv tool install lightcone-cli +git-annex version ``` -It reports the plumbing `repaired` and re-points it at a git-annex -that actually runs — the one that ships with `lc` itself when your -`PATH` has none. +If `lc` runs but `git-annex` does not, uv's tool directory is not on +your `PATH` — run `uv tool update-shell` and open a new shell. Running +`lc` through `uvx` puts nothing on your `PATH` at all, so a plain +`git add` cannot work that way. This failure is deliberately loud. `lc init` sets `filter.annex.required=true` in every project precisely because diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index 61f3be8..15b884e 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -18,8 +18,6 @@ from __future__ import annotations -import os -import shlex from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path @@ -105,198 +103,40 @@ def ignore_rule(directory: Path, path: str) -> str | None: # ============================================================================= -# The annex plumbing: how the researcher's own git reaches git-annex +# The annex filter: a missing git-annex must not be silent # ============================================================================= -# -# `git annex init` wires the repository to a bare `git-annex`, resolved -# from whatever PATH the *researcher's* git runs under — the filter -# drivers in `.git/config` and four hooks. When their shell has no -# git-annex (lc itself always does — it bundles one), a plain `git add` -# prints an error, exits 0, and stages the raw bytes into git history: -# a 2 GB dataset lands in git proper, silently, on every clone forever. -# `filter.annex.required=true` turns that into git's own hard failure, -# and — only where PATH would not resolve git-annex — the plumbing is -# pinned to an absolute path instead, so the ordinary `git add` the docs -# promise keeps working. Measured: with the filter pinned, `git add` of -# annexed content stages a pointer with no git-annex on PATH at all. - - -#: The marker `git annex init` leaves in every hook it writes — and the -#: ownership test: git-annex itself rewrites only hooks that carry it, -#: and so does lc. A hook without it is the user's, never touched. -_HOOK_MARKER = "automatically configured by git-annex" - -#: What each hook runs — git-annex's own hook bodies, mirrored, with the -#: annex spelling left open: ``git annex`` in stock form, an absolute -#: path when pinned. -_HOOKS = { - "pre-commit": "{annex} pre-commit .", - "post-checkout": "{annex} smudge --update", - "post-merge": "{annex} smudge --update", - "post-receive": ( - "if {annex} post-receive --help >/dev/null 2>&1; then {annex} post-receive; fi" - ), -} - -#: The filter drivers `git annex init` configures, same convention -#: (stock spells these ``git-annex``, without the space). The -#: smudge/clean pair stays beside ``process``: git falls back to it when -#: the process filter cannot start, and a fallback that resolves -#: git-annex differently from the filter it stands in for would route -#: the same add two ways. -_FILTERS = { - "filter.annex.process": "{annex} filter-process", - "filter.annex.smudge": "{annex} smudge -- %f", - "filter.annex.clean": "{annex} smudge --clean -- %f", -} - - -def pinned_annex(directory: Path) -> Path | None: - """Read which absolute git-annex the filter is pinned to, if any. - - The record convergence's repair rule works from: the first word of - ``filter.annex.process``. Stock plumbing spells it bare - (``git-annex filter-process``), which is not a pin. - Args: - directory: A directory inside the repository. - - Returns: - The pinned executable, or ``None`` for stock or absent plumbing. - """ - try: - words = shlex.split(_config(directory, "filter.annex.process")) - except ValueError: - return None - if words and Path(words[0]).is_absolute(): - return Path(words[0]) - return None +def require_annex_filter(directory: Path) -> None: + """Make git refuse, rather than corrupt, when the annex filter cannot run. -def annex_runs(path: Path) -> bool: - """Whether *path* is still an executable — the repair-rule probe. - - A pinned engine can be moved or its uv cache pruned; a pin that no - longer runs is what turns ``required=true``'s loud failure back into - a working ``git add`` on the next ``lc init``. - - Args: - path: The recorded git-annex executable. - - Returns: - True if the file exists and is executable. - """ - return path.is_file() and os.access(path, os.X_OK) - - -def converge_annex_plumbing(directory: Path, pin: Path | None) -> None: - """Write the plumbing that lets the researcher's own git find git-annex. - - ``filter.annex.required=true`` always: without it a ``git add`` whose - filter cannot start exits 0 and stages raw bytes into git history — - silent corruption; with it, git refuses loudly and ``lc init`` is the - remedy. The filter drivers and the four hooks are written in stock - form (PATH-resolved, as ``git annex init`` writes them) or pinned to - one absolute executable — the caller decides which, because only it - knows what the researcher's shell resolves. Hooks keep git-annex's - own shebang and marker comment, so git-annex still recognises them - as machinery; a hook without the marker is the user's and is left - alone. + ``git annex init`` wires ``filter.annex.*`` to a bare ``git-annex``, + resolved from the ``PATH`` of whichever git runs — and the + researcher's shell, unlike lc's own environment, may hold none. + Measured: a ``git add`` whose clean filter cannot start prints an + error, **exits 0**, and stages the raw bytes into git history, so a + 2 GB input lands in git proper on every clone forever. This flag is + what turns that into git's own hard failure instead + (``fatal: : clean filter 'annex' failed``), which is the whole + difference between a loud stop and silent corruption. Args: directory: A directory inside the repository. - pin: The absolute git-annex to pin everything to, or ``None`` - for the stock, PATH-resolved form. """ _git(["config", "filter.annex.required", "true"], cwd=directory) - config, hook = _annex_spellings(pin) - for key, command in _FILTERS.items(): - _git(["config", key, command.format(annex=config)], cwd=directory) - # A hooks directory that is not there holds nothing git would run, so - # there is nothing to converge in it — `git init` always creates one. - hooks = _hooks_dir(directory) - if not hooks.is_dir(): - return - marker = _HOOK_MARKER + (", pinned by lightcone-cli" if pin else "") - for name, command in _HOOKS.items(): - path = hooks / name - if path.exists() and _HOOK_MARKER not in path.read_text(): - continue - path.write_text(f"#!/bin/sh\n# {marker}\n{command.format(annex=hook)}\n") - path.chmod(0o755) - - -def annex_plumbing_current(directory: Path, *, stock_ok: bool) -> bool: - """Whether the plumbing needs no write on this host. - - Current means: ``required=true`` is set, and the filter drivers and - every hook lc owns resolve git-annex the same one way — a stock, - PATH-resolved spelling where the shell can honour it, or a pin that - still runs. A pin that works stays current even when PATH could now - serve (both work, and rewriting would flip-flop between a tool - install and a uvx cache); only a broken state reports otherwise. + + +def annex_filter_required(directory: Path) -> bool: + """Whether the repository already carries that flag. Args: directory: A directory inside the repository. - stock_ok: Whether PATH-resolved plumbing works for the - researcher's shell — the caller's :func:`~lightcone.engine. - project.ambient_annex` question. Returns: - True if a convergence would write nothing. - """ - if _config(directory, "filter.annex.required") != "true": - return False - pin = pinned_annex(directory) - if pin is None and not stock_ok: - return False - if pin is not None and not annex_runs(pin): - return False - config, hook = _annex_spellings(pin) - if any(_config(directory, k) != c.format(annex=config) for k, c in _FILTERS.items()): - return False - hooks = _hooks_dir(directory) - if not hooks.is_dir(): - return True # nothing git would run — the write path's judgment too - for name, command in _HOOKS.items(): - path = hooks / name - text = path.read_text() if path.exists() else "" - if text and _HOOK_MARKER not in text: - continue # the user's hook — theirs in the write path too - if command.format(annex=hook) not in text: - return False - return True - - -def _annex_spellings(pin: Path | None) -> tuple[str, str]: - """How the config and the hooks spell git-annex, stock or pinned. - - Stock differs between the two sites — the filters call the - executable (``git-annex``), the hooks go through git's dispatch - (``git annex``) — while a pin is one absolute path at both. - """ - if pin is None: - return "git-annex", "git annex" - quoted = shlex.quote(pin.as_posix()) - return quoted, quoted - - -def _config(directory: Path, key: str) -> str: - """Read one repository config value, empty when unset.""" - found = project._run(["git", "config", "--get", key], cwd=directory) - return str(found.stdout or "").strip() if found.returncode == 0 else "" - - -def _hooks_dir(directory: Path) -> Path: - """Where this repository's hooks live. - - Asked of git rather than spelled ``.git/hooks``: in a linked - worktree ``.git`` is a file, hooks live in the common directory, and - ``core.hooksPath`` can move them anywhere — all questions - ``--git-path`` already answers. + True if ``filter.annex.required`` is set to ``true``. """ - hooks = _git(["rev-parse", "--git-path", "hooks"], cwd=directory).strip() - return (directory / hooks).resolve() + set_to = _ask(["config", "--get", "filter.annex.required"], cwd=directory) + return (set_to or "").strip() == "true" # ============================================================================= diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 6950e10..1637b48 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -7,7 +7,6 @@ import re import shutil import subprocess -import sys import uuid from collections.abc import Callable, Sequence from dataclasses import asdict, dataclass, field @@ -313,62 +312,6 @@ def require_git_annex() -> None: ) -def bundled_annex() -> Path | None: - """Locate the git-annex this engine carries, beside its interpreter. - - Every install of lightcone-cli links the git-annex wheel's entry - points into the same bin directory as ``lc`` and its python, so the - sibling of :data:`sys.executable` is the one place the engine's own - git-annex is by construction — never ambient ``PATH``, which is the - researcher's and answers a different question. - - Returns: - The executable, or ``None`` on a broken install. - """ - found = shutil.which("git-annex", path=str(_engine_bin())) - return Path(found) if found else None - - -def _engine_bin() -> Path: - """The bin directory the engine's executables live in. - - The *parent* of ``sys.executable``, resolved as a directory only: in - a venv the python file itself is a symlink to the managed - interpreter, and following it first would answer with a directory - that holds no entry point of ours. - """ - return Path(sys.executable).parent.resolve() - - -def ambient_annex() -> Path | None: - """Locate the git-annex the researcher's *own* shell would find. - - ``PATH`` minus the engine's bin directory: launchers front that - directory on lc's environment (uvx, ``uv run``), so a naive - ``which`` from inside lc always finds the bundled copy — while the - shell the researcher types ``git add`` into may hold nothing of the - sort. This is the probe that decides whether stock, PATH-resolved - annex plumbing actually works for them. - - Returns: - The executable as the shell would resolve it, or ``None``. - """ - ours = _engine_bin() - kept = [ - entry - for entry in os.environ.get("PATH", "").split(os.pathsep) - if entry and Path(entry).resolve() != ours - ] - found = shutil.which("git-annex", path=os.pathsep.join(kept)) - # Directories are compared, never the file's own symlink target: a - # tool install's `~/.local/bin/git-annex` points into the engine's - # venv, and it is still genuinely the shell's — PATH keeps resolving - # it for exactly as long as that install exists. - if found is None or Path(found).parent.resolve() == ours: - return None - return Path(found) - - def uv_prefix(directory: Path, *, sync: bool) -> list[str]: """Build the ``uv run`` hop that pins a command to a project. @@ -465,11 +408,18 @@ def _converge_dataset(c: _Converger, directory: Path) -> None: c.item(".git", _in_repository(directory), lambda: dataset.init_git(directory)) # After the item above, so a fresh project has a repository to annex. # `annexed` is read *before* the item applies: in write mode the init - # runs inline, and the plumbing item below must still know whether - # this run created the annex (created) or found one (repaired). + # runs inline, and the filter item below must still know whether this + # run created the annex (created) or found one (repaired). annexed = _can_ask_git(directory) and dataset.is_annexed(directory) c.item("git-annex", annexed, lambda: dataset.init_annex(directory)) - _converge_annex_plumbing(c, directory, present=annexed) + # The one thing `git annex init` does not set, and the reason it + # matters is in `dataset.require_annex_filter`. + c.item( + "annex-filter", + annexed, + lambda: dataset.require_annex_filter(directory), + is_current=lambda: dataset.annex_filter_required(directory), + ) attributes = directory / ".gitattributes" # Read before the repair, not after: the check is on the text a repair # would leave behind, and check mode never writes one. @@ -498,60 +448,6 @@ def _converge_dataset(c: _Converger, directory: Path) -> None: _converge_committable(c, directory) -def _converge_annex_plumbing(c: _Converger, directory: Path, *, present: bool) -> None: - """The plumbing that lets the researcher's *own* git find git-annex. - - ``git annex init`` wires the filter drivers and four hooks to a bare - ``git-annex``, resolved from the PATH of whichever git runs — and the - researcher's shell, unlike lc's environment, may hold no git-annex at - all (the engine invoked through uvx puts nothing on it). Measured: - a ``git add`` whose clean filter cannot start prints an error, exits - 0, and stages the raw bytes into git history. So convergence owns the - plumbing: ``filter.annex.required=true`` always, making that failure - git's own hard refusal with ``lc init`` as the remedy — and where the - shell would not resolve git-annex, the filter and hooks are pinned to - the engine's bundled executable by absolute path, so the ordinary - ``git add`` the docs promise keeps working. Where the shell *does* - resolve one, the stock form is kept: it already works, and it outlives - a prunable engine path. - - The repair rule is asymmetric on purpose. A pin that still runs is - left alone even when it is not this engine's copy or PATH could now - serve — both work, and rewriting would flip-flop between a tool - install and a uvx cache. Only a broken state is rewritten: toward - stock when the shell resolves git-annex, else to this engine's copy. - - Args: - c: The converger. - directory: The project root. - present: Whether the annex existed before this convergence — - plumbing written over a fresh ``git annex init`` is created, - plumbing brought up to date on an adopted annex is repaired. - """ - recorded = dataset.pinned_annex(directory) if present else None - keep = recorded if recorded is not None and dataset.annex_runs(recorded) else None - ambient = ambient_annex() - target = keep or (None if ambient is not None else bundled_annex()) - if keep is None and ambient is None and target is None: - # Near-unreachable — require_git_annex found one somewhere — but a - # missing bundled copy must degrade to advice, not a wrong write. - c.warn( - "the annex plumbing was left as git-annex wrote it: no git-annex " - "on PATH for the shell's own `git add` to find, and lc's bundled " - "copy is missing beside its interpreter. " - "`uv tool install --force lightcone-cli` repairs the install." - ) - return - c.item( - "annex-plumbing", - present, - lambda: dataset.converge_annex_plumbing(directory, target), - is_current=lambda: dataset.annex_plumbing_current( - directory, stock_ok=ambient is not None - ), - ) - - def _converge_committable(c: _Converger, directory: Path) -> None: """Refuse to call a project converged while its outputs are unignorable. diff --git a/tests/conftest.py b/tests/conftest.py index 0e72a5b..cf4642a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -102,9 +102,9 @@ def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: # rather than on disk because `.git` is a *file* in a linked # worktree, so there is nowhere inside it to leave a marker. annexed: set[Path] = set() - # What `git config` has been told, per repository — the annex - # plumbing converges through config reads and writes, and a fake - # that forgot them would report the same items repaired forever. + # What `git config` has been told, per repository — the annex filter + # converges through a config write and a config read, and a fake that + # forgot the write would report the same item repaired forever. config: dict[tuple[Path, str], str] = {} def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: @@ -119,9 +119,7 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: elif argv[:2] == ["uv", "sync"]: (project / ".venv" / "bin").mkdir(parents=True, exist_ok=True) elif argv[:2] == ["git", "init"]: - # With the hooks directory, as the real `git init` makes it — - # the annex plumbing writes hook files into it. - (cwd / ".git" / "hooks").mkdir(parents=True, exist_ok=True) + (cwd / ".git").mkdir(exist_ok=True) elif argv[:3] == ["git", "annex", "init"]: annexed.add(_repo(cwd)) elif argv[:2] == ["git", "config"] and argv[-1] == "annex.uuid": @@ -132,10 +130,6 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: return MagicMock(returncode=code, stdout=(value or "") + "\n", stderr="") elif argv[:2] == ["git", "config"] and len(argv) == 4 and not argv[2].startswith("-"): config[(_repo(cwd), argv[2])] = argv[3] - elif argv[:3] == ["git", "rev-parse", "--git-path"]: - return MagicMock( - returncode=0, stdout=str(_repo(cwd) / ".git" / argv[3]) + "\n", stderr="" - ) elif argv[:2] == ["git", "check-ignore"]: return _fake_check_ignore(cwd, argv[-1]) return MagicMock(returncode=0, stdout="", stderr="") diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 591bc47..d5a0ff5 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -500,7 +500,7 @@ def test_the_worker_and_the_shim_are_never_console_scripts() -> None: assert entry.value.startswith("lightcone.cli") or entry.value in theirs, entry -# ---- the annex plumbing, against a real shell ------------------------------- +# ---- the annex filter, against a real shell --------------------------------- def _researcher_shell(tmp_path: Path) -> dict[str, str] | None: @@ -536,37 +536,15 @@ def no_annex_shell(tmp_path: Path) -> dict[str, str]: return env -def test_pinned_plumbing_annexes_content_with_no_git_annex_on_path( - repo: Path, no_annex_shell: dict[str, str] -) -> None: - """The pin's whole claim, measured: with the filter and hooks pinned - to the engine's git-annex by absolute path, the ordinary `git add` - the docs promise stages an annex pointer — on a shell that could not - resolve git-annex at all.""" - annex = project.bundled_annex() - assert annex is not None, "the test environment carries the git-annex wheel" - dataset.converge_annex_plumbing(repo, annex) - (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) - - added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") - assert added.returncode == 0, added.stderr - staged = _shell_git(repo, no_annex_shell, "cat-file", "-p", ":data/catalog.fits") - assert staged.stdout.startswith("/annex/objects/"), "a pointer, not the bytes" - - # The commit exercises the pinned pre-commit hook the same way. - committed = _shell_git(repo, no_annex_shell, "commit", "-m", "input data") - assert committed.returncode == 0, committed.stderr - assert _annexed(repo, repo / "data" / "catalog.fits") - - def test_required_true_makes_the_missing_filter_loud_not_silent( repo: Path, no_annex_shell: dict[str, str] ) -> None: - """The safety net on its own: stock plumbing plus `required=true`, - and a shell without git-annex gets git's own hard refusal — nothing - staged, exit nonzero — instead of raw bytes in git history. `lc init` - is the remedy; it pins the plumbing on exactly such a host.""" - dataset._git(["config", "filter.annex.required", "true"], cwd=repo) + """The whole claim, measured: with the flag set, a shell that cannot + resolve git-annex gets git's own hard refusal — nothing staged, exit + nonzero — instead of raw bytes in git history.""" + assert not dataset.annex_filter_required(repo) + dataset.require_annex_filter(repo) + assert dataset.annex_filter_required(repo) (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") @@ -580,10 +558,10 @@ def test_required_true_makes_the_missing_filter_loud_not_silent( def test_stock_plumbing_without_required_stages_raw_bytes_silently( repo: Path, no_annex_shell: dict[str, str] ) -> None: - """The hazard the plumbing convergence exists for, recorded as - measured: as `git annex init` leaves a repository, a shell without - git-annex prints an error, *exits 0*, and stages the raw bytes into - git history — a 2 GB dataset in git proper, on every clone, forever. + """The hazard the flag exists for, recorded as measured: as + `git annex init` leaves a repository, a shell without git-annex + prints an error, *exits 0*, and stages the raw bytes into git + history — a 2 GB dataset in git proper, on every clone, forever. If this ever starts failing, git changed the behavior and the `required=true` net is worth re-examining.""" (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) @@ -591,57 +569,13 @@ def test_stock_plumbing_without_required_stages_raw_bytes_silently( added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") assert added.returncode == 0 - assert "git-annex: command not found" in added.stderr + # Not the shell's exact wording: bash says "command not found" where + # dash — /bin/sh on Debian and Ubuntu — says only "not found". + assert "not found" in added.stderr staged = _shell_git(repo, no_annex_shell, "cat-file", "-p", ":data/catalog.fits") assert not staged.stdout.startswith("/annex/objects/") -def test_stock_spellings_mirror_what_git_annex_itself_writes(repo: Path) -> None: - """Mirrored, never invented: with only the `required` flag added to - what `git annex init` wrote, the plumbing is already current — so an - upstream change to the stock filter or hook spelling fails here, - rather than reporting every existing project repaired forever.""" - dataset._git(["config", "filter.annex.required", "true"], cwd=repo) - assert dataset.annex_plumbing_current(repo, stock_ok=True) - # And stock is not current where the shell cannot resolve it. - assert not dataset.annex_plumbing_current(repo, stock_ok=False) - - -def test_the_pin_is_recorded_and_its_health_probed(repo: Path, tmp_path: Path) -> None: - """The repair rule works from the recorded path: readable back out of - the filter config, current while it runs, drift the moment it dies.""" - annex = tmp_path / "engine" / "git-annex" - annex.parent.mkdir() - annex.write_text("#!/bin/sh\nexit 0\n") - annex.chmod(0o755) - - dataset.converge_annex_plumbing(repo, annex) - - assert dataset.pinned_annex(repo) == annex - assert dataset.annex_plumbing_current(repo, stock_ok=False) - annex.unlink() - assert not dataset.annex_plumbing_current(repo, stock_ok=True) - - -def test_repair_to_stock_rewrites_a_pinned_repository(repo: Path, tmp_path: Path) -> None: - """The way back: a broken pin on a host whose PATH resolves git-annex - is returned to the stock form, byte-for-byte what `git annex init` - writes plus the required flag — and git-annex still recognises the - hooks as its own.""" - annex = tmp_path / "engine" / "git-annex" - annex.parent.mkdir() - annex.write_text("#!/bin/sh\nexit 0\n") - annex.chmod(0o755) - dataset.converge_annex_plumbing(repo, annex) - - dataset.converge_annex_plumbing(repo, None) - - assert dataset.pinned_annex(repo) is None - assert dataset.annex_plumbing_current(repo, stock_ok=True) - hook = (repo / ".git" / "hooks" / "pre-commit").read_text() - assert "git annex pre-commit ." in hook - assert str(annex) not in hook - # ---- who last wrote a path ------------------------------------------------- diff --git a/tests/test_project.py b/tests/test_project.py index 02bc68a..1c6d91f 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -10,7 +10,7 @@ import pytest from conftest import probes, uv_calls -from lightcone.engine import templates +from lightcone.engine import dataset, templates from lightcone.engine.project import ( SPEC_FILENAME, ConvergenceReport, @@ -216,8 +216,7 @@ def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: state git does not clone. `.venv` is git-ignored and rebuilt from the lock, `git clone` of an annexed repository leaves the annex uninitialized until someone runs `git annex init`, and the annex - plumbing lives in `.git` (config and hooks), which a clone starts - fresh.""" + filter is a `.git/config` entry, which a clone starts fresh.""" project = tmp_path / "proj" converge(project) @@ -232,7 +231,7 @@ def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: (clone / ".git").mkdir() report = converge(clone, write=False) - assert report.created == ["git-annex", "annex-plumbing", ".venv"], report.created + assert report.created == ["git-annex", "annex-filter", ".venv"], report.created def test_converge_repairs_a_missing_piece(tmp_path: Path) -> None: @@ -507,35 +506,15 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: converge(tmp_path / "proj") -# ---- the annex plumbing ---------------------------------------------------- +# ---- the annex filter ------------------------------------------------------ # -# `git annex init` wires the filter and hooks to a bare `git-annex`, -# resolved from the PATH of whichever git runs — and the researcher's -# shell may hold none (the engine invoked through uvx puts nothing on -# it), where a plain `git add` exits 0 and stages raw bytes into git -# history. Convergence owns the plumbing: `required=true` always, and a -# pin to the engine's own executable exactly where PATH would not serve. - - -def _engine_annex(tmp_path: Path, name: str = "engine") -> Path: - """A stand-in for the bundled git-annex: a real executable file, so - the repair rule's exists-and-runs probe has something to answer about.""" - bindir = tmp_path / name - bindir.mkdir(exist_ok=True) - annex = bindir / "git-annex" - annex.write_text("#!/bin/sh\nexit 0\n") - annex.chmod(0o755) - return annex - - -def _annex_resolution( - monkeypatch: pytest.MonkeyPatch, *, ambient: Path | None, bundled: Path | None -) -> None: - """Pin what the two PATH questions answer, host-independently.""" - from lightcone.engine import project as project_mod - - monkeypatch.setattr(project_mod, "ambient_annex", lambda: ambient) - monkeypatch.setattr(project_mod, "bundled_annex", lambda: bundled) +# `git annex init` wires the filter to a bare `git-annex`, resolved from +# the PATH of whichever git runs — and the researcher's shell may hold +# none (the engine invoked through uvx puts nothing on it), where a plain +# `git add` prints an error, exits 0, and stages raw bytes into git +# history. `filter.annex.required=true` makes git refuse instead. What +# that refusal actually looks like is measured in test_dataset.py, +# against a real annex and a real shell. def _config_writes(tools: list[list[str]]) -> dict[str, str]: @@ -547,201 +526,71 @@ def _config_writes(tools: list[list[str]]) -> dict[str, str]: } -def test_ambient_annex_never_answers_with_the_engines_own_copy( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The probe asks what the *researcher's* shell resolves, and lc's own - environment always fronts the engine's bin (uv run, uvx) — so that - directory must be blind-spotted even when it is all PATH holds. In a - venv `sys.executable` is a symlink: resolving it as a file first made - the engine's directory stop matching itself, and the probe reported - the bundled copy as the shell's.""" - import sys - - from lightcone.engine import project as project_mod - - monkeypatch.setenv("PATH", str(Path(sys.executable).parent)) - assert project_mod.ambient_annex() is None +def _annexed_but_unconverged(directory: Path) -> None: + """A repository as `git annex init` alone leaves it: annexed, and + carrying none of the config lc adds on top.""" + directory.mkdir(parents=True, exist_ok=True) + dataset.init_git(directory) + dataset.init_annex(directory) -def test_bundled_annex_is_the_engines_sibling() -> None: - """The suite runs where the git-annex wheel is installed, so the - engine's own copy resolves beside its interpreter — the path every - pin is made of.""" - from lightcone.engine import project as project_mod - - assert project_mod.bundled_annex() == project_mod._engine_bin() / "git-annex" - - -def test_a_shell_without_git_annex_gets_the_plumbing_pinned( - tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch -) -> None: - """The uvx case: nothing lands on the researcher's PATH, so their own - `git add` must reach the engine's git-annex by absolute path — filter - drivers and all four hooks, the post-receive guard included.""" - annex = _engine_annex(tmp_path) - _annex_resolution(monkeypatch, ambient=None, bundled=annex) +def test_convergence_requires_the_annex_filter(tmp_path: Path, tools: list[list[str]]) -> None: + """The one thing `git annex init` does not set, and the only thing + this convergence writes — no filter driver is rewritten and no hook + is touched, so what git-annex configured stays exactly as it wrote it.""" project = tmp_path / "proj" report = converge(project) - assert "annex-plumbing" in report.created + assert "annex-filter" in report.created written = _config_writes(tools) assert written["filter.annex.required"] == "true" - assert written["filter.annex.process"] == f"{annex} filter-process" - assert written["filter.annex.smudge"] == f"{annex} smudge -- %f" - assert written["filter.annex.clean"] == f"{annex} smudge --clean -- %f" - hooks = project / ".git" / "hooks" - assert f"{annex} pre-commit ." in (hooks / "pre-commit").read_text() - assert f"{annex} smudge --update" in (hooks / "post-checkout").read_text() - assert f"{annex} smudge --update" in (hooks / "post-merge").read_text() - receive = (hooks / "post-receive").read_text() - assert f"if {annex} post-receive --help" in receive - assert "git annex" not in receive - for name in ("pre-commit", "post-checkout", "post-merge", "post-receive"): - text = (hooks / name).read_text() - assert text.startswith("#!/bin/sh\n") - assert "automatically configured by git-annex, pinned by lightcone-cli" in text - - -def test_a_shell_with_git_annex_keeps_stock_plumbing_plus_the_required_net( - tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch -) -> None: - """Where the researcher's PATH already resolves git-annex, the stock - form works and outlives any engine path — only `required=true` is - added, so a PATH that later loses git-annex fails loudly instead of - staging raw bytes.""" - _annex_resolution( - monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path) - ) - project = tmp_path / "proj" + assert [key for key in written if key.startswith("filter.annex.")] == [ + "filter.annex.required" + ] - report = converge(project) - assert "annex-plumbing" in report.created - written = _config_writes(tools) - assert written["filter.annex.required"] == "true" - assert written["filter.annex.process"] == "git-annex filter-process" - hook = (project / ".git" / "hooks" / "pre-commit").read_text() - assert "git annex pre-commit ." in hook - - -@pytest.mark.parametrize("ambient", [None, Path("/usr/bin/git-annex")]) -def test_converged_plumbing_reconverges_unchanged( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ambient: Path | None -) -> None: +def test_a_converged_project_reconverges_unchanged(tmp_path: Path) -> None: + """The flag is read back, so a second run reports it unchanged rather + than repairing it on every invocation forever.""" project = tmp_path / "proj" - _annex_resolution(monkeypatch, ambient=ambient, bundled=_engine_annex(tmp_path)) converge(project) report = converge(project) + assert "annex-filter" in report.unchanged assert report.converged - assert "annex-plumbing" in report.unchanged - - -def test_a_dead_pin_is_repaired_to_the_current_engine( - tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch -) -> None: - """A uv cache prune or a moved engine breaks the recorded path; - `required=true` makes that loud, and the next convergence points the - plumbing at the engine that is actually here.""" - project = tmp_path / "proj" - old = _engine_annex(tmp_path, "old-engine") - _annex_resolution(monkeypatch, ambient=None, bundled=old) - converge(project) - old.unlink() - new = _engine_annex(tmp_path, "new-engine") - _annex_resolution(monkeypatch, ambient=None, bundled=new) - - report = converge(project) - assert "annex-plumbing" in report.repaired - assert _config_writes(tools)["filter.annex.process"] == f"{new} filter-process" - assert f"{new} pre-commit ." in (project / ".git" / "hooks" / "pre-commit").read_text() - -def test_a_dead_pin_returns_to_stock_when_the_shell_has_git_annex( - tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +def test_an_adopted_annex_without_the_flag_is_repaired( + tmp_path: Path, tools: list[list[str]] ) -> None: - """Repair goes toward the durable answer: a PATH-resolved git-annex - outlives any engine install, so a broken pin on a host that has one - is put back the way `git annex init` writes it.""" + """A repository annexed before lc ever saw it: the annex is already + there, so the flag is *repaired* onto it rather than created.""" project = tmp_path / "proj" - old = _engine_annex(tmp_path, "old-engine") - _annex_resolution(monkeypatch, ambient=None, bundled=old) - converge(project) - old.unlink() - _annex_resolution( - monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path) - ) + _annexed_but_unconverged(project) + tools.clear() report = converge(project) - assert "annex-plumbing" in report.repaired - assert _config_writes(tools)["filter.annex.process"] == "git-annex filter-process" - assert "git annex pre-commit ." in (project / ".git" / "hooks" / "pre-commit").read_text() - + assert "annex-filter" in report.repaired + assert _config_writes(tools)["filter.annex.required"] == "true" -def test_a_working_pin_survives_git_annex_appearing_on_path( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Both forms work, so rewriting would only flip-flop between a tool - install and a uvx cache — a healthy state is never rewritten.""" - project = tmp_path / "proj" - pinned = _engine_annex(tmp_path) - _annex_resolution(monkeypatch, ambient=None, bundled=pinned) - converge(project) - _annex_resolution( - monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path, "other") - ) - - report = converge(project) - - assert report.converged - assert "annex-plumbing" in report.unchanged - assert f"{pinned} pre-commit ." in (project / ".git" / "hooks" / "pre-commit").read_text() - -def test_check_mode_reports_plumbing_drift_and_writes_nothing( - tmp_path: Path, tools: list[list[str]], monkeypatch: pytest.MonkeyPatch +def test_check_mode_reports_the_missing_flag_and_writes_nothing( + tmp_path: Path, tools: list[list[str]] ) -> None: - """The PATH that satisfied stock plumbing at init can stop resolving - git-annex later; `--check` must say so — and, as everywhere, without - fixing it.""" + """Check mode asks the same question and answers it without writing: + the drift is reported, and no `git config` write is issued.""" project = tmp_path / "proj" - _annex_resolution( - monkeypatch, ambient=Path("/usr/bin/git-annex"), bundled=_engine_annex(tmp_path) - ) - converge(project) - _annex_resolution(monkeypatch, ambient=None, bundled=_engine_annex(tmp_path)) - hooks_before = (project / ".git" / "hooks" / "pre-commit").read_bytes() + _annexed_but_unconverged(project) tools.clear() report = converge(project, write=False) - assert "annex-plumbing" in report.repaired + assert "annex-filter" in report.repaired assert not report.converged assert _config_writes(tools) == {} - assert (project / ".git" / "hooks" / "pre-commit").read_bytes() == hooks_before - - -def test_a_users_own_hook_is_never_touched( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """The marker `git annex init` leaves is the ownership test — a hook - without it is the user's, skipped by the write and by the drift - check alike, exactly as git-annex itself treats it.""" - project = tmp_path / "proj" - _annex_resolution(monkeypatch, ambient=None, bundled=_engine_annex(tmp_path)) - converge(project) - theirs = "#!/bin/sh\nblack --check .\n" - (project / ".git" / "hooks" / "pre-commit").write_text(theirs) - - report = converge(project) - - assert report.converged - assert (project / ".git" / "hooks" / "pre-commit").read_text() == theirs # ---- refusals ------------------------------------------------------------- From a83edffb74711018be95ce87fea655ecd42ae4a9 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 17:00:58 +0200 Subject: [PATCH 3/5] Read the annex flag from the repository, not the merged config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `annex_filter_required` asked `git config --get`, which reads the merged system + global + local config, while `require_annex_filter` writes the repository-local one. Measured: with `filter.annex.required = true` in `~/.gitconfig`, `--get` returns true/exit 0 while `--local --get` exits 1 — so on such a host `lc init` reported the item unchanged and never wrote the key into the project at all. The protection then vanished the moment that repository was used under another HOME (a clone, CI, a container), with `--check` still calling it converged. `--type=bool` fixes the mirror case: `1`, `yes`, `on` and a valueless key are all true to the filter, and the string compare read them as drift, so `--check` failed on a repository that was already correct. Both directions are pinned by tests against real git, and both were mutation-checked — four of the five fail without the fix. Docs: the refusal is wider than "a `git add` refuses". Measured on a project holding committed annexed content, `git status`, `git diff` and `git checkout` all abort with exit 128 too, so the troubleshooting entry and the init and architecture pages now say the project reads as broken until git-annex is back, which is the intended trade. CLAUDE.md records the external check: the rule neighbours agree on is route-through-a-filter ⇒ set `required`. git-lfs routes and sets it; `datalad create` does neither, and a plain `git add` in a DataLad dataset stages raw bytes into git even with git-annex working, its safeguard being procedural (`datalad save`). lc promised the researcher their own `git add`, so lc owns the failure mode that promise creates. Also `GIT_CONFIG_NOSYSTEM` in the researcher-shell fixture, so a site-wide `/etc/gitconfig` cannot decide those measurements. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016MWNSzVFUMmieXA9ayPtbL --- CLAUDE.md | 15 +++++++++++- docs/architecture.md | 3 ++- docs/cli/init.md | 6 ++++- docs/user/troubleshooting.md | 8 +++++++ src/lightcone/engine/dataset.py | 20 +++++++++++++--- tests/conftest.py | 7 ++++-- tests/test_dataset.py | 41 ++++++++++++++++++++++++++++++++- 7 files changed, 91 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 66cb791..f3e20db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -561,7 +561,20 @@ instead of silent corruption. path into a repository and rewrites no filter driver or hook: how git dispatches git-annex stays `PATH` resolution, which is git-annex's own design and git-lfs's too (it writes `filter.lfs.required = true` beside -`PATH`-relative drivers and has never baked a path). Pinning the filter +`PATH`-relative drivers and has never baked a path). + +The rule the neighbours agree on is **route through a filter ⇒ set +`required`**, and both halves were measured. git-lfs routes and sets +it. `datalad create` does *neither*: no `required`, and no +`* filter=annex` in its `.gitattributes` (just `* annex.backend=MD5E` +and `**/.git* annex.largefiles=nothing`) — so a plain `git add big.bin` +in a DataLad dataset stages 200 000 raw bytes into git *even with +git-annex working*, and the safeguard is procedural: use `datalad +save`. lc made the opposite promise — the researcher types the git they +already know — so lc owns the failure mode that promise creates. The +honest cost, also measured: once a project holds committed annexed +content, the refusal covers `git status`, `git diff` and `git checkout` +too, not just `git add`. Pinning the filter and the four hooks to the engine's bundled executable was implemented and reverted before merge: it writes durable state pointing at a prunable `uvx` cache, `core.hooksPath` can send the hooks into a diff --git a/docs/architecture.md b/docs/architecture.md index f1610ab..af9a3bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -107,7 +107,8 @@ ordinary `git add` / `git commit`. That ordinary `git add` dispatches git-annex from the *researcher's* `PATH`, and a shell that cannot resolve it stages the raw bytes into git history while exiting 0 — so `lc init` sets -`filter.annex.required=true`, which makes git refuse loudly instead. +`filter.annex.required=true`, which makes git refuse loudly instead +(every filtered command, not only `git add`). Getting git-annex onto that `PATH` is the install's job, not the repository's: `uv tool install lightcone-cli` puts it there alongside `lc`. diff --git a/docs/cli/init.md b/docs/cli/init.md index 26aa672..ff36709 100644 --- a/docs/cli/init.md +++ b/docs/cli/init.md @@ -89,7 +89,11 @@ Inside `.git`, convergence sets one configuration key — reported as the shell cannot find git-annex prints an error, **exits 0, and stages the raw bytes into git history** — a 2 GB dataset in git proper, on every clone, forever. With it, the same situation is a hard, loud - failure and nothing is staged. + failure and nothing is staged. Once the project holds committed + annexed content that refusal covers every command that must run the + filter, `git status` and `git diff` included — a project you cannot + use until git-annex is back, rather than one that silently absorbed + your data. That is the only thing `lc init` adds to what `git annex init` wrote. How git finds git-annex is still ordinary `PATH` resolution, which is diff --git a/docs/user/troubleshooting.md b/docs/user/troubleshooting.md index de6d10e..73f0b2a 100644 --- a/docs/user/troubleshooting.md +++ b/docs/user/troubleshooting.md @@ -129,6 +129,14 @@ which is the point: without `filter.annex.required=true` — which `lc init` sets — git would have exited 0 and committed the raw bytes into history instead. +Once a project holds committed annexed content, this is not limited to +`git add`. Any command that has to run the filter over that content +stops the same way, `git status`, `git diff` and `git checkout` +included — so the whole project reads as broken until git-annex is back +on your `PATH`. That is the intended shape of the failure: a repository +you cannot use is recoverable in one command, and one that quietly +absorbed a multi-gigabyte file is not. + `git-annex` ships with `lc`, so a tool install puts both on your `PATH`: ```bash diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index 15b884e..d9632cf 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -127,15 +127,29 @@ def require_annex_filter(directory: Path) -> None: def annex_filter_required(directory: Path) -> bool: - """Whether the repository already carries that flag. + """Whether *this repository* already carries that flag. + + ``--local`` because the write is repository-local, and a probe that + read the merged config would answer for the *host* instead: a user + who once set ``filter.annex.required`` in ``~/.gitconfig`` would have + every project report converged while its own ``.git/config`` carried + nothing, leaving the protection behind the moment that repository is + used under another ``HOME`` — a clone, CI, a container. ``--type=bool`` + because git's booleans are not one spelling: ``1``, ``yes``, ``on`` + and a valueless key all mean true to the filter, and reading them as + drift would rewrite a repository that was already correct. Args: directory: A directory inside the repository. Returns: - True if ``filter.annex.required`` is set to ``true``. + True if ``filter.annex.required`` is set, in this repository, to + anything git reads as true. """ - set_to = _ask(["config", "--get", "filter.annex.required"], cwd=directory) + set_to = _ask( + ["config", "--local", "--get", "--type=bool", "filter.annex.required"], + cwd=directory, + ) return (set_to or "").strip() == "true" diff --git a/tests/conftest.py b/tests/conftest.py index cf4642a..04a0188 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -124,8 +124,11 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: annexed.add(_repo(cwd)) elif argv[:2] == ["git", "config"] and argv[-1] == "annex.uuid": return MagicMock(returncode=0 if _repo(cwd) in annexed else 1) - elif argv[:3] == ["git", "config", "--get"]: - value = config.get((_repo(cwd), argv[3])) + elif argv[:3] == ["git", "config", "--local"] and "--get" in argv: + # The key is last; the flags in between are git's own reading + # conventions (`--type=bool`), which the store does not model + # because the engine only ever writes one value. + value = config.get((_repo(cwd), argv[-1])) code = 0 if value is not None else 1 return MagicMock(returncode=code, stdout=(value or "") + "\n", stderr="") elif argv[:2] == ["git", "config"] and len(argv) == 4 and not argv[2].startswith("-"): diff --git a/tests/test_dataset.py b/tests/test_dataset.py index d5a0ff5..c2e5418 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -516,7 +516,14 @@ def _researcher_shell(tmp_path: Path) -> dict[str, str] | None: return None if any((d / "git-annex").exists() for d in dirs): return None - return {"PATH": ":".join(str(d) for d in dirs), "HOME": str(tmp_path)} + # `HOME` takes the global config out of play; `GIT_CONFIG_NOSYSTEM` + # does the same for `/etc/gitconfig`, where a site-wide + # `filter.annex.required` would otherwise decide these outcomes. + return { + "PATH": ":".join(str(d) for d in dirs), + "HOME": str(tmp_path), + "GIT_CONFIG_NOSYSTEM": "1", + } def _shell_git(repo: Path, env: dict[str, str], *argv: str) -> subprocess.CompletedProcess[str]: @@ -555,6 +562,38 @@ def test_required_true_makes_the_missing_filter_loud_not_silent( assert staged.stdout.strip() == "" +def test_the_flag_is_read_from_the_repository_never_the_users_global_config( + repo: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The write is repository-local, so the read must be too. A user who + once set the flag in `~/.gitconfig` would otherwise have every project + report converged while its own `.git/config` carried nothing — and + since this is local state a clone never receives, the protection would + vanish the moment the repository moved to CI or another account.""" + global_config = tmp_path / "gitconfig" + global_config.write_text('[filter "annex"]\n\trequired = true\n') + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(global_config)) + + assert not dataset.annex_filter_required(repo) + + dataset.require_annex_filter(repo) + assert dataset.annex_filter_required(repo) + assert "required = true" in (repo / ".git" / "config").read_text() + + +@pytest.mark.parametrize("spelling", ["true", "1", "yes", "on"]) +def test_any_spelling_git_reads_as_true_is_not_reported_as_drift( + repo: Path, spelling: str +) -> None: + """git's booleans are not one string. A repository someone configured + by hand as `required = 1` is fully protected — reading it as drift + would rewrite a repository that was already correct, and report every + `lc init --check` on it as unconverged.""" + dataset._git(["config", "filter.annex.required", spelling], cwd=repo) + + assert dataset.annex_filter_required(repo) + + def test_stock_plumbing_without_required_stages_raw_bytes_silently( repo: Path, no_annex_shell: dict[str, str] ) -> None: From de33fbce67691ab89c38cf5324c9986997e95a27 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 18:00:30 +0200 Subject: [PATCH 4/5] =?UTF-8?q?Assert=20the=20refusal,=20not=20git's=20wor?= =?UTF-8?q?ding=20=E2=80=94=20macOS=20refuses=20differently?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tests workflow has been red on macOS since the branch opened, on both measured tests, for one underlying reason: Linux git and macOS git handle a filter that will not start differently. Linux git reports the failed handshake, falls back to smudge/clean, and with `required` unset **exits 0 and stages the raw bytes** — the hazard this whole change exists to close. macOS git instead dies on the pkt-line read with `fatal: the remote end hung up unexpectedly`, exit 128, whether or not the flag is set. So: - The loud-refusal test asserts the two facts that are the claim — nonzero exit, nothing staged — instead of pinning `clean filter 'annex' failed`, which is Linux's wording. (Same lesson as the bash/dash one, one level up.) - The silent-passthrough test is Linux-only by `skipif`, because the behaviour it records is Linux's. macOS being safe there is a git implementation detail that can change on any release, not a guarantee — the flag is what makes it one, so this is a reason to keep the flag unscoped, not to narrow it. CLAUDE.md records the asymmetry beside the storage invariants: the hazard is Linux's, the guarantee is ours, and CI running both platforms is what surfaced it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016MWNSzVFUMmieXA9ayPtbL --- CLAUDE.md | 11 +++++++++++ tests/test_dataset.py | 20 ++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f3e20db..246ff4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -563,6 +563,17 @@ dispatches git-annex stays `PATH` resolution, which is git-annex's own design and git-lfs's too (it writes `filter.lfs.required = true` beside `PATH`-relative drivers and has never baked a path). +**The hazard is Linux's, the guarantee is ours.** The exits-0-and-stages-raw-bytes +behaviour is what Linux git does; macOS git aborts the same handshake +outright (exit 128, `fatal: the remote end hung up unexpectedly`) and +would have been safe without the flag — found by CI, which is why the +suite runs on both. So `test_stock_plumbing_without_required_stages_raw_bytes_silently` +is Linux-only by `skipif`, and the loud-refusal test asserts the facts +(nonzero exit, nothing staged) rather than either platform's wording. +Do not read macOS's accident as a reason to scope the flag: an +implementation detail that can change on the next git release is not a +guarantee, and Linux is where the venues are. + The rule the neighbours agree on is **route through a filter ⇒ set `required`**, and both halves were measured. git-lfs routes and sets it. `datalad create` does *neither*: no `required`, and no diff --git a/tests/test_dataset.py b/tests/test_dataset.py index c2e5418..8272844 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -11,6 +11,7 @@ import shutil import subprocess +import sys from pathlib import Path import pytest @@ -556,8 +557,12 @@ def test_required_true_makes_the_missing_filter_loud_not_silent( added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") + # The facts, not git's wording: Linux git reports the failed handshake + # and then `fatal: : clean filter 'annex' failed`, while macOS + # git dies on the pkt-line read with `fatal: the remote end hung up + # unexpectedly`. Both refuse, which is the whole claim. assert added.returncode != 0 - assert "clean filter 'annex' failed" in added.stderr + assert "annex" in added.stderr staged = _shell_git(repo, no_annex_shell, "diff", "--cached", "--name-only") assert staged.stdout.strip() == "" @@ -594,6 +599,12 @@ def test_any_spelling_git_reads_as_true_is_not_reported_as_drift( assert dataset.annex_filter_required(repo) +@pytest.mark.skipif( + sys.platform == "darwin", + reason="macOS git aborts the filter handshake outright (exit 128, " + "'the remote end hung up unexpectedly'), so the silent-passthrough " + "hazard recorded here is the Linux behaviour", +) def test_stock_plumbing_without_required_stages_raw_bytes_silently( repo: Path, no_annex_shell: dict[str, str] ) -> None: @@ -602,7 +613,12 @@ def test_stock_plumbing_without_required_stages_raw_bytes_silently( prints an error, *exits 0*, and stages the raw bytes into git history — a 2 GB dataset in git proper, on every clone, forever. If this ever starts failing, git changed the behavior and the - `required=true` net is worth re-examining.""" + `required=true` net is worth re-examining. + + Linux-only, and that asymmetry is the point rather than a gap: macOS + git happens to refuse the same situation on its own, so there the + flag changes nothing — but a platform accident is not a guarantee, + and the flag is what makes the refusal one on every host.""" (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) added = _shell_git(repo, no_annex_shell, "add", "data/catalog.fits") From cc2cc4badfd1b6c0f747f761825e20bc14be4904 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 18:34:41 +0200 Subject: [PATCH 5/5] Rename require_annex_filter to set_annex_filter_required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `require_*` means "assert this precondition or refuse" everywhere else in the engine — `require_uv`, `require_git`, `require_git_annex`, `require_committer` all raise ProjectError and mutate nothing. This one writes config, so the prefix said the opposite of what it does, and both return None, so nothing would have caught the misreading. `set_annex_filter_required` also pairs by name with the `annex_filter_required` probe. The two have to agree about config scope, which is exactly where the local-versus-merged bug came from, so a name that makes a future divergence obvious is worth having. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016MWNSzVFUMmieXA9ayPtbL --- docs/api/dataset.md | 2 +- src/lightcone/engine/dataset.py | 2 +- src/lightcone/engine/project.py | 4 ++-- tests/test_dataset.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/api/dataset.md b/docs/api/dataset.md index a6362ac..53df884 100644 --- a/docs/api/dataset.md +++ b/docs/api/dataset.md @@ -20,7 +20,7 @@ Source: `src/lightcone/engine/dataset.py` (+ | `last_writer(root, dir)` | Who last touched an output's directory — the foreign-write question. Answers "cannot say" as empty, never an error. | | `require_committer(root)` | Refuses a repository with no git identity, before any recipe spends time. Asked as `git var`, the question a commit itself asks. | | `dataset_id(root)` | The DataLad dataset UUID, read via `git config -f`. | -| `require_annex_filter(root)` | Set `filter.annex.required=true`, so a `git add` that cannot reach git-annex fails loudly instead of staging raw bytes. | +| `set_annex_filter_required(root)` | Set `filter.annex.required=true`, so a `git add` that cannot reach git-annex fails loudly instead of staging raw bytes. | | `annex_filter_required(root)` | Whether that flag is already set — `lc init --check`'s question. | ## What must stay true diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index d9632cf..8b3d91e 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -107,7 +107,7 @@ def ignore_rule(directory: Path, path: str) -> str | None: # ============================================================================= -def require_annex_filter(directory: Path) -> None: +def set_annex_filter_required(directory: Path) -> None: """Make git refuse, rather than corrupt, when the annex filter cannot run. ``git annex init`` wires ``filter.annex.*`` to a bare ``git-annex``, diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 1637b48..c907216 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -413,11 +413,11 @@ def _converge_dataset(c: _Converger, directory: Path) -> None: annexed = _can_ask_git(directory) and dataset.is_annexed(directory) c.item("git-annex", annexed, lambda: dataset.init_annex(directory)) # The one thing `git annex init` does not set, and the reason it - # matters is in `dataset.require_annex_filter`. + # matters is in `dataset.set_annex_filter_required`. c.item( "annex-filter", annexed, - lambda: dataset.require_annex_filter(directory), + lambda: dataset.set_annex_filter_required(directory), is_current=lambda: dataset.annex_filter_required(directory), ) attributes = directory / ".gitattributes" diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 8272844..9e4dc1c 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -551,7 +551,7 @@ def test_required_true_makes_the_missing_filter_loud_not_silent( resolve git-annex gets git's own hard refusal — nothing staged, exit nonzero — instead of raw bytes in git history.""" assert not dataset.annex_filter_required(repo) - dataset.require_annex_filter(repo) + dataset.set_annex_filter_required(repo) assert dataset.annex_filter_required(repo) (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 4096) @@ -581,7 +581,7 @@ def test_the_flag_is_read_from_the_repository_never_the_users_global_config( assert not dataset.annex_filter_required(repo) - dataset.require_annex_filter(repo) + dataset.set_annex_filter_required(repo) assert dataset.annex_filter_required(repo) assert "required = true" in (repo / ".git" / "config").read_text()