diff --git a/.github/workflows/sandbox-macos.yml b/.github/workflows/sandbox-macos.yml new file mode 100644 index 00000000..c2c9afa6 --- /dev/null +++ b/.github/workflows/sandbox-macos.yml @@ -0,0 +1,34 @@ +name: sandbox-macos + +# The Seatbelt realization of the lc sandbox can only be verified on +# macOS — this smoke is its single verification path (the dev loop and +# the main test matrix run Linux/Landlock). + +on: + push: + branches: [main] + paths: + - "src/lightcone/engine/sandbox/**" + - "src/lightcone/_sandbox_exec.py" + - "tests/test_seatbelt.py" + - ".github/workflows/sandbox-macos.yml" + pull_request: + paths: + - "src/lightcone/engine/sandbox/**" + - "src/lightcone/_sandbox_exec.py" + - "tests/test_seatbelt.py" + - ".github/workflows/sandbox-macos.yml" + workflow_dispatch: + +jobs: + seatbelt-smoke: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - name: Install + run: uv sync --group dev + - name: Seatbelt enforcement smoke + run: uv run pytest tests/test_seatbelt.py -v -m darwin + - name: Sandbox unit tests (mechanism-neutral) + run: uv run pytest tests/test_sandbox_policy.py tests/test_sandbox_shim.py tests/test_sandbox_denial.py -q diff --git a/CLAUDE.md b/CLAUDE.md index 12a3d49c..17c138b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,153 +2,155 @@ ## Project Overview -**lightcone-cli** is Lightcone Research's execution layer for ASTRA (Agentic Schema for Transparent Research Analysis). It ships the `lc` executable — an agent-agnostic CLI; it bundles no agent-specific skills, hooks, or plugins. - -- **ASTRA** = pure specification: schema, validation, prior insights & findings, evidence verification, helpers, minimal CLI -- **lightcone-cli** = execution layer: project scaffolding, **Snakemake-based execution**, container builds - -lightcone-cli depends on ASTRA. The `astra` CLI handles spec operations; the `lc` CLI handles execution. +**lightcone-cli** is Lightcone Research's execution layer for ASTRA +(Agentic Schema for Transparent Research Analysis). It ships the `lc` +executable — an agent-agnostic CLI; it bundles no agent-specific +skills, hooks, or plugins. + +- **ASTRA** = pure specification: schema, validation, helpers, minimal + CLI. Carries analysis structure only (inputs/outputs/recipes/ + decisions/universes) — **never** environment or sandbox information. +- **lightcone-cli** = execution layer: the uv-based environment model, + Snakemake-based execution, the OS sandbox, the podman container + hatch, and per-output provenance manifests. + +The normative design is `docs/design/execution-environment.md` (v6.1); +the implementation deviates from it only where recorded (ASTRA carries +no container/sandbox keys; `PYTHONPYCACHEPREFIX` amendment; local-only +venue scope with podman as the only container backend). ### Namespace contract -`lightcone-cli` ships the `lightcone.*` namespace via PEP 420 implicit namespace packages. **`src/lightcone/` must not contain an `__init__.py`** — that would turn the namespace into a regular package and break coexistence with future sibling distributions (`lightcone-ui`, etc.). - -Any new `lightcone-*` package must: - -1. Use src-layout (`src/lightcone//…`). -2. Not create `src/lightcone/__init__.py`. -3. Ship only its own subpackage under `src/lightcone//`. +`lightcone-cli` ships the `lightcone.*` namespace via PEP 420 implicit +namespace packages. **`src/lightcone/` must not contain an +`__init__.py`** — that would break coexistence with future sibling +distributions. Any new `lightcone-*` package must use src-layout, not +create `src/lightcone/__init__.py`, and ship only its own subpackage. ## Architecture -The execution layer is a thin shim over Snakemake. The integrity layer (per-output content-addressed manifests) is the only thing we own substantively. - -``` -astra.yaml ── snakefile generator ──> .lightcone/Snakefile - │ - snakemake (CLI subprocess) - │ - ┌───────────────────────────────┼───────────────────────────────┐ - │ │ │ │ │ - DAG resolution staleness cluster submission container exec conda - (Snakemake) (mtime+code) (slurm plugin) (apptainer/docker) - │ - └─── per-rule run: block: shell() recipe + write_manifest() - │ - results///... - results///.lightcone-manifest.json -``` - -**What Snakemake owns** (we do not write code for any of this): DAG construction, topological execution, parallelism (`--cores`, `--jobs`), cluster submission (`snakemake-executor-plugin-slurm`), per-rule resources, profiles, dry-run, DAG visualization, staleness detection (`--rerun-triggers`), locking, log capture, retry, container runtime invocation. - -**What we own**: a Snakefile generator, the manifest layer (write/read/verify), a status walker, and a verify routine. - -## Repository Structure - -``` -src/lightcone/ # namespace — NO __init__.py -├── cli/ # Click surface -│ ├── __init__.py # exposes main() -│ └── commands.py # init, run, status, verify, build -├── engine/ # execution substrate — Snakemake-based -│ ├── __init__.py -│ ├── manifest.py # write_manifest, sha256_dir, code_version — the integrity layer -│ ├── snakefile.py # generates .lightcone/Snakefile from astra.yaml -│ ├── container.py # Content-addressed container builds (Docker, podman-hpc, apptainer) -│ ├── cloudbuild.py # GCP Cloud Build backend (JupyterHub deployments; no local OCI runtime) -│ ├── status.py # Manifest-driven status walker (no Snakemake import) -│ ├── verify.py # Recompute hashes; validate provenance chain -│ ├── tree.py # Sub-analysis tree traversal (kept from before) -│ ├── validation.py # Post-materialization output shape checks -│ └── site_registry.py # Known HPC site defaults (Perlmutter, etc.) - -evals/ # Agentic eval: prompt.md + tasks// seed files; - # driven by .github/workflows/eval.yml (no Python harness) -tests/ # pytest — mirrors src/ structure -pyproject.toml # hatchling + hatch-vcs, ASTRA + Snakemake as deps -``` +Three owned layers over a Snakemake core (see `docs/architecture.md` +for the full picture): + +1. **Environment** — a project is `pyproject.toml` + `uv.lock` + + `.python-version`; uv is the only substrate. Mode is *derived*: + declaring `[tool.lightcone.image]` (or `Containerfile.extra`) is the + escalation into containerized mode. `env_version` (lock ‖ interpreter + pin ‖ install settings ‖ system layer) sits inside every output's + `code_version`. The launcher (`lightcone/launcher.py`) delegates + execution verbs to the project-locked engine — frozen interface: + argv passthrough + `LC_DELEGATED=1`. +2. **Integrity** — `.lightcone-manifest.json` per output + (SCHEMA_VERSION 2): code/env/data versions, input chain, git state, + runtime attestation, image identity, hermeticity. The manifest is a + declared Snakemake output; a failing recipe writes none. +3. **Hermeticity** — every recipe/probe executes through the + `ExecBoundary` (`engine/boundary.py` → `engine/sandbox/`): Landlock + on Linux (incl. in-container), Seatbelt on macOS; declared-set + policy (own-output RW, project+inputs RO, env + versioned utility + allowlist + ELF loaders exec, fresh per-recipe HOME/XDG/ + PYTHONPYCACHEPREFIX). The shim is `lightcone/_sandbox_exec.py` + (stdlib-only; exit 97 = setup failure). Manifests record the + *applied* enforcement; downgrades are announced, never silent. + +**Images** (`engine/image/`): one-TOML-table surface, Modal-inspired +internals — declaration → deterministic render (fixed layering, apt +before `uv sync`, offline ENV in the final stage only) → tag +`lc-env-` as a pure function of rendered text + pyproject + +uv.lock → podman build with pointed error mapping → digest-pinned +full-stack run (`--net=none`, `--userns=keep-id`, entrypoint cleared). +Project code never enters an image; code edits never move the tag. + +**Execution flow**: `lc materialize` → launcher converges/delegates → +`snakefile.generate()` (recipes rendered but **never wrapped**; typed +`RuleJob` cfg) → `snakemake --executor dask` on a run-scoped +LocalCluster → `run_rule()` worker sequence: pre-gate (env_version +recomputed vs baked) → env check (`uv sync --check` / image identity +assert) → boundary exec with offline overlay → post-gate → +`write_manifest`. + +## CLI surface + +- `lc init` — idempotent uv-native scaffold (pyproject with the engine + locked in, `.python-version`, `uv lock` + sync, AGENTS.md stanza); + refuses an authored root Containerfile; `--check`/`--json`. +- `lc materialize [outputs…]` — execute; `--require-sandbox[=declared-fs]`, + `--no-sandbox` (flags reach workers via env, never cfg). +- `lc run [cmd…]` — the probe verb: arbitrary commands in byte-for-byte + the recipe environment (sandboxed, tmp-only writes); rename guard for + output ids; `--sandbox-debug`; never builds images. +- `lc status` — offline; 3-line header (mode/image/sandbox) + per-output + states incl. `pre_migration`; blast-radius line. +- `lc verify` — tamper/chain checks + notes (unsandboxed, dirty_tree, + pre_migration). +- `lc build` — containerized mode only (direct = explanatory no-op). +- `lc export wrroc` — RO-Crate bundle. ## Development Commands ```bash -uv sync --group dev # installs pytest, ruff, mypy -uv run pytest +uv sync --group dev +uv run pytest # default: excludes slow/podman marks +uv run pytest -m podman # real podman builds (this machine has podman) uv run ruff check src/ tests/ -uv run mypy src/ +uv run mypy src/ # strict ``` -A `justfile` is available for common tasks — run `just` to see all recipes: - -```bash -just test # run pytest -just lint # ruff + mypy -just docs # build the documentation site -``` - -## Architecture & Data Flow - -``` -astra.yaml ── snakefile.generate() ──> .lightcone/Snakefile + .lightcone/snakefile-config.json - │ - snakemake -s ... -d ... - │ - per-rule run: - │ - shell(recipe) ────────────────► write_manifest() - (in container if container: set; (host-side) - Snakemake handles invocation) - │ - results///data.txt - results///.lightcone-manifest.json -``` - -- `snakefile.generate(project, universes=[...])` reads `astra.yaml`, writes `.lightcone/Snakefile`, and writes a sidecar JSON keyed by `(rule, universe)` containing the recipe text, container image, decisions, and precomputed `code_version`. -- The Snakefile body for each rule is a `run:` block: `shell(params.cfg["recipe"])` then `write_manifest(...)`. -- `code_version = sha256(recipe + container_image + decisions)`. Embedded in the rule's shell command literally so Snakemake's built-in `code` rerun-trigger detects drift. -- `data_version = sha256_dir(output_dir)`. Written into the manifest after the recipe completes; used by `lc verify` to detect tampering. Excludes the manifest file itself and `.snakemake_timestamp`. -- The manifest is a *declared output* of every rule. A missing manifest causes Snakemake to re-run the rule, blocking the agent-faked-file scenario. +A `justfile` covers common tasks (`just test`, `just lint`, +`just docs`). The macOS Seatbelt smoke runs only in +`.github/workflows/sandbox-macos.yml`. ## Key Invariants -**Spec & execution:** -- `astra.yaml` is the single source of truth — all inputs, outputs, recipes, decisions, containers -- Output paths are always `results///` for root and inline sub-analyses; `/results///` for path-rooted sub-analyses -- Container image hashes are deterministic: SHA256(Containerfile + dependency files) → `lc--` -- The Snakefile and snakefile-config.json are regenerated on every `lc run` — never edit them by hand - -**Integrity:** -- Every materialized output has `/.lightcone-manifest.json` recording code_version, data_version, container, recipe, decisions, input_versions, git_sha, lc_version, host -- `lc verify` recomputes data_version and walks the chain; failures surface as `tampered_data`, `broken_chain`, or `missing_manifest` -- `lc status` reads only manifests — works offline, no Snakemake or DB needed - -**CLI surface:** -- `lc init` — idempotently converge a project (astra.yaml, .gitignore, .lightcone/, results/, universes/, Containerfile, MyST report template); `--check` reports drift without writing, `--json` emits the report -- `lc run [outputs...]` — generate Snakefile, invoke snakemake -- `lc status` — manifest-driven status report -- `lc verify` — chain integrity check -- `lc build` — pre-build container images from Containerfiles - -Global config (`~/.lightcone/config.yaml`) is auto-created with defaults on first invocation. - -## Extending the Codebase - -| To... | Read | Key patterns | +- `astra.yaml` = analysis structure only; legacy `container:` keys are + ignored. The environment is the uv project + `[tool.lightcone.*]` + (a **closed** surface — unknown keys are refusals). +- `.lightcone/Snakefile` + `snakefile-config.json` are regenerated + every run; `.lightcone/image/` is the machine-local build record + (gitignored). +- `code_version = sha256({recipe, decisions, env_version, + writable_project})` — computed only via `manifest.code_version()`, + shared by generator and status. +- Per-output sandbox escalation lives in pyproject: + `[tool.lightcone.sandbox] writable-project = [""]` — + hashes into that output's `code_version`, not `env_version`. +- Manifest filename `.lightcone-manifest.json` is fixed; changing + semantics means bumping `SCHEMA_VERSION` and the golden field-list + test. `sha256_dir` excludes the manifest and `.snakemake_timestamp`. +- Engine constants (base/uv digests, `DEFAULT_PYTHON`) live in + `engine/image/constants.py` and change only with an engine release. +- Golden tests pin: env_version fingerprints (`test_environment.py`), + rendered Containerfiles (`tests/goldens/`, regen with + `--regen-goldens`), the manifest v2 field list, and the frozen + delegation interface. + +## Extending + +| To… | Read | Pattern | |---|---|---| -| Add a CLI command | `src/lightcone/cli/commands.py` | `@main.command()`, project discovery via `_project_root()` | -| Change manifest semantics | `src/lightcone/engine/manifest.py` + `tests/test_manifest.py` | Bump `SCHEMA_VERSION`; add a test | -| Change Snakefile shape | `src/lightcone/engine/snakefile.py` + `tests/test_snakefile.py` | Includes a `snakemake -n` parse test | -| Add container features | `src/lightcone/engine/container.py` | `compute_image_tag()`, build/resolve functions | +| Add a CLI verb | `cli/commands.py` + `launcher.py` | decide tool-env vs delegated (TOOL_ENV_VERBS) | +| Change identity semantics | `engine/environment.py`, `engine/manifest.py` | bump goldens consciously, same commit | +| Change the sandbox policy | `engine/sandbox/policy.py` | bump `EXEC_ALLOWLIST_VERSION`; add an enforcement test | +| Change the image | `engine/image/definition.py`/`render.py` | regen Containerfile goldens; podman smoke | +| Add a container backend / venue | `engine/image/builder.py` protocol, `engine/boundary.py` | implement the protocol; never fork call sites | ## Test Patterns -- `tests/test_manifest.py` — pure-function tests for the integrity layer -- `tests/test_snakefile.py` — generator tests; final test runs `snakemake -n` on the output -- `tests/test_status.py` / `tests/test_verify.py` — end-to-end against a tmp project -- `tests/test_cli.py` — Click `CliRunner().invoke(main, [...])` patterns +- Fixture projects come from `tests/conftest.py::make_project` + (deterministic bytes — identity goldens hash them). +- Landlock enforcement tests (`test_sandbox_enforcement.py`) run + unprivileged but place projects under `$HOME`, NOT `tmp_path` — the + policy grants `/tmp` blanket-RW, which would mask denials. +- `lc init` tests fake the uv seam (`commands._run_uv`) — never real + resolution in unit tests. +- Real-subsystem tests are opt-in marks: `slow` (LocalCluster), + `podman` (real builds), `darwin` (Seatbelt smoke on macOS CI). ## Conventions -- Ruff for linting (E, F, I, N, W, UP), line length 100, target Python 3.11 -- mypy strict mode with `namespace_packages = true`, `explicit_package_bases = true` -- Manifest filename is fixed: `.lightcone-manifest.json` (don't change without bumping `SCHEMA_VERSION`) -- Snakemake's `directory()` outputs require excluding `.snakemake_timestamp` from the hash +- Ruff (E, F, I, N, W, UP), line length 100, target Python 3.11; mypy + strict with `namespace_packages = true`. +- Errors are the interface: refusals carry the exact fix (the denial + UX, base-contract messages, `lc build` pointers). Never a raw log + or a silent fallback. +- The manifest records what actually ran — never what should have. diff --git a/README.md b/README.md index faaf15dc..51022dc4 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,12 @@ provenance. ## Quick Start ```bash -uv tool install lightcone-cli # or: pip install lightcone-cli +uv tool install lightcone-cli # uv is the only prerequisite lc init my-analysis cd my-analysis +uv add numpy astropy # dependencies live in the lock # describe your analysis in astra.yaml, then: -lc run +lc materialize ``` ASTRA specs are plain, structured YAML — they work well hand-written or @@ -31,8 +32,10 @@ drafted with any AI coding assistant. ## Capabilities - **Multiverse analysis** — define methodological decisions with multiple options; `lc` runs your analysis across all defensible paths automatically +- **Locked environments** — uv is the only substrate: the exact interpreter, every dependency, and the engine itself are pinned in the project's lock, and that identity is recorded in every output +- **Sandboxed execution** — every recipe runs inside an OS sandbox (Landlock/Seatbelt) restricted to its declared inputs and outputs; each manifest records the enforcement that actually ran - **Provenance integrity** — every output gets a content-addressed manifest; `lc verify` detects tampering or broken chains -- **HPC-ready execution** — Snakemake-backed DAG dispatch with SLURM and container support (Docker, Podman, Apptainer) out of the box +- **A container hatch, not a container tax** — projects that need system dependencies (R, TeX, CUDA userlands) declare one TOML table; `lc` generates a content-addressed podman image from the lock — never a hand-written Containerfile, and code edits never trigger rebuilds - **Reproducible publishing** — `lc export wrroc` emits a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) bundle ready for Zenodo or WorkflowHub → [Full documentation](https://docs.lightconeresearch.org) diff --git a/docs/api/assets.md b/docs/api/assets.md deleted file mode 100644 index 39815f8a..00000000 --- a/docs/api/assets.md +++ /dev/null @@ -1,5 +0,0 @@ -# lightcone.engine.assets (removed) - -This module was the Dagster asset factory. It no longer exists. The Snakemake -generator that replaced it lives at -[engine/snakefile](snakefile.md). diff --git a/docs/api/cli.md b/docs/api/cli.md deleted file mode 100644 index df679953..00000000 --- a/docs/api/cli.md +++ /dev/null @@ -1,88 +0,0 @@ -# lightcone.cli.commands - -The Click surface. Defined in `src/lightcone/cli/commands.py`. Six -public commands: `init`, `run`, `status`, `verify`, `build`, `export`. - -The user-facing reference is in [CLI Overview](../cli/index.md). This -page is a tour of the module internals. - -## Entry point - -```python -@click.group() -@click.version_option(package_name="lightcone-cli") -@click.pass_context -def main(ctx: click.Context) -> None: - ctx.ensure_object(dict) - _ensure_global_config() # auto-create ~/.lightcone/config.yaml with defaults -``` - -`main` is exposed as `lightcone.cli.main` (re-exported from -`lightcone.cli.__init__`) and is the entry point declared in -`pyproject.toml::project.scripts`: - -```toml -[project.scripts] -lc = "lightcone.cli:main" -``` - -## Helpers - -### `_config_path() → Path` - -Returns `~/.lightcone/config.yaml`. Used by `_ensure_global_config()`, -which the `main` group calls to create the file with defaults -(`container: {runtime: auto}`) on first invocation. - -### `_project_root(start: Path | None = None) → Path` - -Walks up from `start` (or `cwd`) looking for `astra.yaml`. Raises -`click.ClickException` if none found. Used by `run`, `status`, `verify`, -`build`. - -### `_target_for(project: Path, output_id: str, universe: str) → str` - -Translate an `output_id` (or qualified `.`) into -the Snakemake target path that materializes it — specifically the -manifest file `results///.lightcone-manifest.json`. -Raises `click.ClickException` if the id is unknown or ambiguous. - -### `_run_snakemake(cmd, *, env, scratch_root, verbose)` - -Spawn `snakemake` and forward the run's narrative output: lines the -executor plugin prefixes with the sentinel -(`lightcone.engine.runner.SENTINEL`) stream to the terminal with the -prefix stripped; everything else (DAG chatter, job stats) is dropped -unless `verbose`. stderr is tailed into a bounded ring buffer and, on -failure, dumped to `snakemake-stderr-.log` under the scratch -root. Returns the exit code. - -### `_status_label(s: str) → str` - -Map a status literal to the Rich-formatted display label: - -| Status | Display | -|--------|---------| -| `ok` | `[green]✓ ok[/green]` | -| `stale` | `[yellow]✸ stale[/yellow]` | -| `missing` | `[red]✗ miss[/red]` | -| `alias` | `[dim]→ alias[/dim]` | - -## Boilerplate text - -`_CONTAINERFILE_TEMPLATE`, `_REQUIREMENTS`, `_GITIGNORE_BASE`, -`_GITIGNORE_APPEND`, `_MYST_YML`, and `_INDEX_MD_BODY` are multi-line -strings written at `lc init` time (the spec boilerplate itself comes -from astra's boilerplate helper). Edit them to change what new -projects look like. - -`init` is a convergence loop, not a one-shot scaffolder: each managed -item is created if missing, offered to an optional -`repair(text) -> str | None` hook otherwise (today only -`_repair_gitignore`, which appends the managed block once), and left -alone when the hook returns `None`. `--check` computes the same report -without writing (exit 1 when not converged); `--json` prints it as -`{converged, created, repaired, unchanged, warnings}`. Warnings carry -problems init can see but must not fix (e.g. a directory `COPY` in the -Containerfile, detected via -`lightcone.engine.container.directory_copy_sources`). diff --git a/docs/api/cloudbuild.md b/docs/api/cloudbuild.md deleted file mode 100644 index 17f0f6a3..00000000 --- a/docs/api/cloudbuild.md +++ /dev/null @@ -1,63 +0,0 @@ -# lightcone.engine.cloudbuild - -Remote image builds through GCP Cloud Build — the build backend for -deployments with no OCI runtime on the host (a JupyterHub user pod on -GKE). Pure `urllib` REST against the metadata server, GCS, the Cloud -Build API, and the Docker Registry v2 API; no SDK dependency, no -stored credentials, no git remote required. - -Source: `src/lightcone/engine/cloudbuild.py`. - -## Deployment contract - -Env vars injected into every user pod by the deployment (see the -hub-deploy `lightcone` hub config): - -| Env var | Meaning | -|---------|---------| -| `LIGHTCONE_REGISTRY` | Artifact Registry prefix (`-docker.pkg.dev//`); also names the GCP project builds run in. Declared in `engine.container` (`REGISTRY_ENV`). | -| `LIGHTCONE_BUILD_BUCKET` | GCS bucket for build sources and logs. Its presence (with the registry) selects this backend — `cloudbuild_available()`. | -| `LIGHTCONE_BUILD_SERVICE_ACCOUNT` | Optional dedicated build SA; the deployment grants it registry-writer rights only. | - -Auth is the pod's Workload Identity, spoken to the GCE metadata server -(`_metadata_access_token()`). The pod's identity needs -`cloudbuild.builds.editor`, `iam.serviceAccountUser` on the build SA, -object create/view on the bucket, and `artifactregistry.reader` for -the freshness probe. - -## `ensure_image(project, containerfile_spec, *, project_name, force=False, on_progress=None) → str` - -Make sure the project's image is in the registry; return its ref. -Content-addressed and git-free: - -1. Compute the ref `$LIGHTCONE_REGISTRY/lc-:` — the - same `image_identity()` digest as the local `lc--` - tag, spelled for a registry. -2. `registry_image_exists(ref)` — one HEAD on the Docker Registry v2 - manifest endpoint. Present → done (no build, no upload). `force` - skips this probe. -3. Tar the **staged build context** (`_populate_build_context` — the - exact file set the tag hashes) and upload it to the bucket under a - content-addressed object name. -4. Submit the build (docker builder step, image push, logs to - `gs:///logs` with `GCS_ONLY` — required for custom SAs and - the source of the failure tail), poll to a terminal status. -5. Non-`SUCCESS` → `CloudBuildError` carrying the build-log tail. - -`on_progress(phase, detail)` phases: `cached`, `staging`, then Cloud -Build statuses lowercased (`queued`, `working`, `success`, …). - -## `registry_image_exists(ref) → bool | None` - -`None` — not `False` — when unknowable (no metadata credentials, -registry unreachable), so callers can distinguish "absent, build it" -from "can't tell". Artifact Registry accepts the OAuth2 access token -directly as a Bearer on `/v2/` endpoints. - -## Tests - -`tests/test_cloudbuild.py` mocks the two HTTP seams -(`_metadata_access_token`, `_request`) and exercises the real control -flow: backend selection, freshness probe, staging, submission -(including the custom-SA payload), polling, failure-tail reporting, -and the staged-tarball ↔ hashed-context equivalence. diff --git a/docs/api/container.md b/docs/api/container.md deleted file mode 100644 index dad10864..00000000 --- a/docs/api/container.md +++ /dev/null @@ -1,192 +0,0 @@ -# lightcone.engine.container - -The container layer. Two surfaces: build-time (`compute_image_tag`, -`build_image`, `pull_image`) and run-time wrap (`wrap_recipe`, -`make_image_tag_resolver`). - -Source: `src/lightcone/engine/container.py`. - -## Constants - -| Constant | Value | -|----------|-------| -| `RUNTIMES` | `("podman", "docker", "podman-hpc")` — detection priority order | -| `DEPENDENCY_FILES` | `("requirements.txt", "requirements-dev.txt", "requirements-test.txt", "pyproject.toml", "setup.py", "setup.cfg", "poetry.lock", "Pipfile.lock")` | - -Detection priority is podman before docker for two reasons: it's -rootless (less surprising on shared machines), and the docker probe -includes `docker info` so a stopped daemon doesn't silently win over a -healthy podman. - -## Runtime detection - -### `detect_runtime() → str | None` - -Returns the first usable runtime in `RUNTIMES`. "Usable" means the -binary is on PATH and (for docker) `docker info` succeeds. Returns -`None` if nothing's available. - -### `load_runtime(*, project_path=None) → RuntimeChoice` - -Resolve the runtime to use. Reads `container.runtime` from -`~/.lightcone/config.yaml`: - -- `auto` (default) → first available, else `"none"` with `explicit=False`. - On a site declaring `container_runtime: kubernetes` (a Dask Gateway - deployment), auto resolves to `kubernetes` with no PATH probing. -- `docker | podman | podman-hpc` → explicit; binary must exist or - raises `ContainerBuildError`. -- `kubernetes` → explicit; no binary involved (the worker pod is the - container). -- `none` → explicit opt-out. -- Anything else → `ContainerBuildError`. - -`project_path` is accepted for future per-project overrides but is not -consulted today. - -### `RuntimeChoice` (dataclass) - -```python -@dataclass(frozen=True) -class RuntimeChoice: - runtime: str # docker | podman | podman-hpc | none - explicit: bool # True if pinned, False if `auto` produced this -``` - -`explicit=False` + `runtime="none"` means auto fell back silently. Callers -should warn — that case mismatches the manifest's recorded -`container_image` against what actually executed. - -## Image tag computation - -### `compute_image_tag(project_name, containerfile, project_path) → str` - -Returns `lc--`. The hash covers the -Containerfile contents plus every dependency file from `DEPENDENCY_FILES` -that exists at the project root. - -Sanitization: lowercase + spaces → hyphens. - -### `find_dependency_files(project_path) → list[Path]` - -Sorted list of dependency files actually present. Used by -`compute_image_tag`. - -### `hash_file_contents(files) → str` - -Concatenated SHA-256 hex digest of the listed files. Internal helper. - -### `is_containerfile(spec, project_path) → bool` - -True if `spec` resolves to an existing file (i.e. it's a Containerfile, -not a registry image). - -## Build - -### `build_image(tag, containerfile, context, *, runtime, build_args=None) → ContainerBuildResult` - -Run ` build -t -f [--build-arg …] `. -For `podman-hpc`, also runs `podman-hpc migrate ` so compute nodes -can read the image. Raises `ContainerBuildError` on any failure. - -### `pull_image(image, *, runtime) → None` - -Run ` pull `, then (for podman-hpc) `migrate`. Used by -`lc build` to pre-stage registry images so `lc run` can pass -`--pull=never`. - -### `image_exists_locally(tag, *, runtime) → bool` - -Check the local image store. Routes to `image_exists_podman_hpc(tag)` -for `podman-hpc`, otherwise runs ` image inspect `. - -### `_podman_hpc_migrate(tag)` (private) - -Wraps `podman-hpc migrate`. Raises `ContainerBuildError` on failure. - -## Run-time wrap - -### `wrap_recipe(recipe, *, image, runtime) → str` - -Wrap `recipe` so it executes inside `image` under `runtime`. Returns a -shell-command string for Snakemake's `shell()`. - -No-op cases (`recipe` returned unchanged): - -- `image is None` -- `runtime == "none"` -- `runtime == "kubernetes"` — the Dask worker pod executing the recipe - was started from `image`; wrapping would containerize twice. The - image still flows into `code_version` and the manifest. - -Otherwise produces: - -```bash - run --rm --pull=never \ - -v "$PWD":"$PWD" -w "$PWD" \ - bash -c '' -``` - -`--pull=never` is critical: it sidesteps podman's -`unqualified-search-registries` resolution, which fails for our -content-addressed `lc--` tags. The cost: registry images -have to be pre-pulled by `lc build`. - -The bind mount and `-w "$PWD"` ensure recipes that write to relative -paths land in the project tree. Snakemake invokes us with `cwd=project`, -so `$PWD` is the project root. - -Snakemake placeholders inside `recipe` (`{output[0]}`, `{input.X}`, -`{wildcards.universe}`) are preserved — they substitute through Python's -`str.format` at execution time, after wrapping. - -### `make_image_tag_resolver(project_path, project_name) → Callable` - -Returns a memoizing wrapper around `resolve_image_for_run`. Multiple -outputs typically share a Containerfile; resolving re-hashes the file -plus all dependency files (lockfiles can be megabytes), so caching by -spec string for the lifetime of the caller's loop matters. - -### `resolve_image_for_run(spec, *, project_path, project_name, registry=None) → str | None` - -Translate an `astra.yaml` `container:` value into the image tag the -runtime will execute: - -- `None` / empty → `None` -- Containerfile path → `lc--` (the tag `lc build` would - produce), or `/lc-:` when `registry` is given - (a deployment with a remote builder — same content-addressed - identity, spelled for a registry) -- Anything else → returned as-is - -## Status - -### `get_container_status(spec, project_path, project_name, *, runtime) → ContainerStatus` - -Without building or pulling, return a `ContainerStatus` describing what -would happen. - -### `ContainerStatus` (dataclass) - -```python -@dataclass -class ContainerStatus: - type: str # "none" | "prebuilt" | "build" - image: str | None = None # the tag (always set for "prebuilt"/"build") - exists: bool | None = None # local-store presence (None for "none" runtime) - containerfile: str | None = None # the spec, only set for "build" -``` - -## Exceptions - -### `ContainerBuildError` - -Raised by `build_image`, `pull_image`, `_podman_hpc_migrate`, and -`load_runtime` (configuration errors). Message carries the failing -runtime and stderr. - -## Tests - -`tests/test_container.py` covers detection, image tag computation, -build invocation, recipe wrapping, and the `RuntimeChoice` resolution -matrix. diff --git a/docs/api/dask_cluster.md b/docs/api/dask_cluster.md deleted file mode 100644 index be2aebd2..00000000 --- a/docs/api/dask_cluster.md +++ /dev/null @@ -1,104 +0,0 @@ -# lightcone.engine.dask_cluster - -Cluster lifecycle for `lc run`. One context manager (`cluster_for_run`), -four branches, no service to manage. - -Source: `src/lightcone/engine/dask_cluster.py`. - -## `cluster_for_run(*, verbose=False, worker_image=None, max_workers=None) → Iterator[dict[str, str]]` - -Yields the env overlay the child snakemake needs to reach the cluster -(the executor plugin lives in a different process, so connection info -travels via environment variables). Four branches in priority order: - -1. **`DASK_SCHEDULER_ADDRESS` already set** → yield it as-is. We don't - own the cluster, so we don't tear it down. -2. **`DASK_GATEWAY__ADDRESS` set** (a JupyterHub deployment) → - **create** a run-scoped Dask Gateway cluster with `worker_image` as - its `image` cluster option, scale it adaptively `1..max_workers`, - wait (bounded by `LIGHTCONE_GATEWAY_WORKER_TIMEOUT`, default 600 s) - for the first worker, and shut the cluster down on exit — success - or failure. Yields `{LIGHTCONE_GATEWAY_CLUSTER: }`: Gateway - schedulers speak a `gateway://` comm scheme a bare `Client` cannot - dial, so the executor rejoins by name through the Gateway API. -3. **`SLURM_JOB_ID` set** → start an in-process scheduler bound to the - driver's SLURM hostname (`SLURMD_NODENAME` or `gethostname()`), - then `srun` one `dask worker` per node across the allocation. -4. **None of the above** → `LocalCluster()` sized to the local machine. - -Outside the Gateway branch the scheduler is always in-process, so its -lifetime equals the run's lifetime: no orphaned schedulers if the -driver crashes. On the Gateway branch the same contract is enforced -server-side — create per run, cull on exit (the deployment's idle -timeout is the backstop). Create-per-run is also what makes image -updates seamless: a Gateway cluster's image is fixed at creation. - -The Gateway branch self-provisions the worker environment through the -deployment's **standard `environment` cluster option** (no -lightcone-specific injection needed server-side): the -`DASK_DISTRIBUTED__WORKER__RESOURCES__*` scheduling contract mirrored -from the declared `worker_cores`/`worker_memory` option values, the -driver's `HOME`/`USER`/`LOGNAME` (passwd-less uid-1000 images crash -`getpass.getuser()` without them), and `LIGHTCONE_WORKER_IMAGE` as -manifest ground truth. It also fails fast on two silent-hang failure -modes: zero workers within the timeout (unpullable image, -unschedulable pool), and workers that don't advertise the -`cpus`/`memory` resource contract (a deployment that doesn't expose -the `environment` option). - -## Resource keys - -These string constants form a contract with the executor plugin: - -```python -RESOURCE_CPUS = "cpus" -RESOURCE_MEMORY = "memory" -RESOURCE_GPUS = "gpus" -``` - -Workers must advertise every key the executor may request — Dask -matches by exact key presence. The local-cluster path includes all -three even when the executor doesn't ask, so per-rule -`mem_mb`/`gpus_per_task` rules still schedule on a workstation. - -## Node-shape detection - -`_detect_node_shape()` reads SLURM env vars with sane fallbacks: - -| Resource | Env var | Fallback | -|----------|---------|----------| -| CPUs | `SLURM_CPUS_ON_NODE` | `os.cpu_count()` | -| Memory | `SLURM_MEM_PER_NODE` (MB) | `psutil.virtual_memory().total` if installed; otherwise 0 (advisory; workers won't enforce caps) | -| GPUs | `SLURM_GPUS_ON_NODE` | `0` | - -## SLURM-backed cluster details - -```python -srun --ntasks=$SLURM_NNODES --ntasks-per-node=1 \ - dask worker --nthreads $cpus --nworkers 1 \ - --resources "cpus=N memory=B gpus=G" --no-dashboard -``` - -The `--ntasks-per-node=1` is important: we want one worker per node, -not per CPU. The worker uses `--nthreads` to advertise its parallelism -within the node. - -After spawning workers, the manager opens a temporary `Client(addr)` to -`wait_for_workers(n_workers=nnodes, timeout=120)`. If the workers -haven't connected within two minutes, raise. - -On exit, the manager `terminate()`s the worker subprocess group, waits -up to 10s, then `kill()`s anything still alive. - -## Why no `dask-jobqueue`? - -`dask-jobqueue` would `sbatch` workers from inside an existing job — -fine, but adds dependency and indirection. Since we already require the -user to be inside an allocation (`salloc` / `sbatch`), `srun` is enough -and keeps everything in one process tree. - -## Tests - -`tests/test_dask_cluster.py` covers the three branches and the -resource-advertising contract. The SLURM branch is tested with mocked -`subprocess.Popen` plus a stubbed `Client.wait_for_workers`. diff --git a/docs/api/dask_executor.md b/docs/api/dask_executor.md deleted file mode 100644 index 28f2ae5d..00000000 --- a/docs/api/dask_executor.md +++ /dev/null @@ -1,118 +0,0 @@ -# snakemake_executor_plugin_dask - -Snakemake executor plugin that submits each rule as a `client.submit()` -on a `dask.distributed` cluster. Lives at the top of `src/` because -Snakemake discovers executor plugins through the -`snakemake_executor_plugin_*` package-naming convention. - -Source: `src/snakemake_executor_plugin_dask/`. - -## Module shape - -``` -snakemake_executor_plugin_dask/ -├── __init__.py # plugin metadata + Executor re-export -└── executor.py # DaskExecutor class -``` - -## Plugin metadata - -```python -common_settings = CommonSettings( - job_deploy_sources=True, # send Snakefile + sources to workers - non_local_exec=True, # workers may live elsewhere - implies_no_shared_fs=False, # we *do* assume a shared FS -) -``` - -We assume a shared filesystem because all our workers (local threads, -SLURM nodes) see the project tree the same way. If you change this, -you also need to teach `wrap_recipe()` not to use `$PWD` bind mounts. - -## `DaskExecutor` - -Inherits from `snakemake_interface_executor_plugins.executors.remote.RemoteExecutor`. - -### `__init__(workflow, logger)` - -Imports `dask.distributed` lazily; raises `WorkflowError` if missing -("`pip install distributed`"). Connects via `_connect_client()`: if -`LIGHTCONE_GATEWAY_CLUSTER` is set, rejoins that Dask Gateway cluster -through `Gateway().connect(name)` (with `shutdown_on_close=False` — -the executor is a guest; `lc run` owns the cluster lifecycle); -otherwise dials `DASK_SCHEDULER_ADDRESS` with a bare `Client`. Raises -`WorkflowError` if neither is set — `lc run` is responsible for -setting one. - -### `run_job(job)` - -Translate a Snakemake job to a Dask submission: - -```python -client.submit( - _run_shell, job.format_job_exec(), - resources=_build_resources(job), - pure=False, - key=f"snakejob-{job.name}-{job.jobid}", -) -``` - -`_run_shell` runs `subprocess.run(cmd, shell=True, check=False)` on -the worker and returns `(exit_code, output_block)`. The block is the -child snakemake's sentinel-prefixed lines (see -`lightcone.engine.runner.SENTINEL`), kept verbatim; on a failure that -produced no sentinel line at all (the child died before the rule body -— import error, missing package in the worker image) a bounded raw -tail is sentinel-framed instead so bootstrap failures don't vanish -into worker logs. Returning output through the task result is the only -channel that works uniformly across LocalCluster threads, srun-launched -workers, and Gateway worker pods (whose stdout goes to pod logs). The -recipe is already container-wrapped at Snakefile generation time, so -the worker has no runtime logic of its own. - -### `check_active_jobs(active_jobs)` - -Async generator: for each submitted job, check `future.done()`. Yield -back jobs that are still in flight. For finished jobs, unpack the -result via `_unpack_result` (which also accepts the bare-int result of -a worker running an older lightcone-cli release), write the output -block to stdout — `lc run` filters and forwards it — then: - -- `future.exception() is not None` → `report_job_error(...)` -- exit code `!= 0` → `report_job_error(...)` -- otherwise → `report_job_success(...)` - -### `cancel_jobs(active_jobs)` - -Best-effort `future.cancel()` on each in-flight job. **Does not** close -the `Client` — Snakemake calls `cancel_jobs` for partial cancellations -as well as at shutdown, so closing here would break subsequent -submissions. The client is closed in `shutdown()` exclusively. - -### `shutdown()` - -`self._client.close()` then `super().shutdown()`. - -## Resource translation - -```python -def _build_resources(job) -> dict[str, float]: - res = {} - cpus = job.resources.get("cpus_per_task") or job.threads - if cpus: res["cpus"] = float(cpus) - mem_mb = job.resources.get("mem_mb") - if mem_mb: res["memory"] = float(mem_mb) * 1e6 # MB → bytes - gpus = job.resources.get("gpus_per_task") or job.resources.get("gpus") - if gpus: res["gpus"] = float(gpus) - return res -``` - -Returns `None` if the resulting dict is empty (Dask's "no constraints" -sentinel). Resource keys must match those advertised by workers -(`cpus`, `memory`, `gpus` — see [`engine.dask_cluster`](dask_cluster.md)). - -## Tests - -`tests/test_dask_plugin.py` exercises the `_build_resources` mapping -and the executor's lifecycle (init / run / check / shutdown) against a -local `LocalCluster` fixture. diff --git a/docs/api/index.md b/docs/api/index.md index 491843b1..4a6555e2 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,61 +1,70 @@ -# Python API Reference - -The interesting public surface lives in `lightcone.engine.*`. The CLI -is a thin Click wrapper around these modules. - -## Module map - -| Module | Role | -|--------|------| -| [`lightcone.cli.commands`](cli.md) | Click CLI: `init`, `run`, `build`, `status`, `verify`, `setup`. | -| [`lightcone.engine.manifest`](manifest.md) | Per-output `.lightcone-manifest.json` write/read; `code_version`, `sha256_dir`. The integrity layer. | -| [`lightcone.engine.snakefile`](snakefile.md) | Generate `.lightcone/Snakefile` and `snakefile-config.json` from `astra.yaml`. | -| [`lightcone.engine.container`](container.md) | Runtime detection, content-addressed image tags, `wrap_recipe`. | -| [`lightcone.engine.dask_cluster`](dask_cluster.md) | Cluster lifecycle for `lc run` (local / SLURM / external). | -| [`lightcone.engine.status`](status.md) | Manifest-driven status walker. | -| [`lightcone.engine.verify`](verify.md) | Recompute hashes; walk the input chain. | -| [`lightcone.engine.tree`](tree.md) | Sub-analysis tree helpers — outputs, decisions, `from:` resolution. | -| [`lightcone.engine.validation`](validation.md) | Post-recipe sanity checks (empty dir, all-NaN columns, …). | -| [`snakemake_executor_plugin_dask`](dask_executor.md) | Snakemake executor plugin → `dask.distributed`. | -| `lightcone.engine.site_registry` | Vestigial — no active code path imports it. See [api/site_registry](site_registry.md). | - -## Common entry points - -```python -from pathlib import Path -from lightcone.engine.snakefile import generate, discover_universes -from lightcone.engine.container import load_runtime - -project = Path("my-analysis") -runtime = load_runtime(project_path=project).runtime -universes = discover_universes(project) # ['baseline', 'experiment'] -snakefile, cfg = generate(project, universes=universes, runtime=runtime) -# Now invoke `snakemake -s snakefile -d project --executor dask ...` -``` - -```python -from lightcone.engine.status import get_output_status - -for s in get_output_status(project, universe_id="baseline"): - print(s.status, s.output_id) # 'ok', 'stale', 'missing', or 'alias' -``` - -```python -from lightcone.engine.verify import verify_outputs - -failed = [r for r in verify_outputs(project, universe_id="baseline") if not r.passed] -for r in failed: - print(r.failure, r.output_id, r.detail) -``` - -```python -from lightcone.engine.container import ( - detect_runtime, - compute_image_tag, - build_image, -) - -runtime = detect_runtime() # 'podman' / 'docker' / 'podman-hpc' / None -tag = compute_image_tag("my-project", Path("Containerfile"), Path(".")) -build_image(tag, Path("Containerfile"), Path("."), runtime=runtime) -``` +# Python API + +The `lightcone.*` namespace, module by module. Signatures live in the +source docstrings — this page is the map. (For the subsystem view, see +[Architecture](../architecture.md).) + +## Top level + +| Module | Responsibility | +|---|---| +| `lightcone.launcher` | the tool-env launcher: discover → mode-detect → scrub → converge → delegate (frozen interface: argv + `LC_DELEGATED=1`) | +| `lightcone._sandbox_exec` | the exec shim (`python -m lightcone._sandbox_exec`) — stdlib-only, applies Landlock/Seatbelt between fork and exec; exit 97 = setup failure | +| `lightcone.cli.commands` | the Click surface: `init`, `materialize`, `run`, `status`, `verify`, `build`, `export` | + +## Engine — environment & identity + +| Module | Responsibility | +|---|---| +| `engine.environment` | `Mode`, `EnvironmentSpec`, `load_environment()`, `compute_env_version()`, `scan_lock()` — the single parse point for the closed `[tool.lightcone]` surface | +| `engine.uv_env` | the closed ambient `UV_*` scrub list + the offline overlay | +| `engine.project` | `find_root()` — the `astra.yaml` walk-up | +| `engine.manifest` | `SCHEMA_VERSION`, `code_version()`, `sha256_dir()`, `write_manifest()`, `is_pre_migration()` | +| `engine.attestation` | `capture_runtime_attestation()` — worker-side platform/interpreter/uv/GPU capture | + +## Engine — execution + +| Module | Responsibility | +|---|---| +| `engine.snakefile` | `generate()` — astra.yaml → Snakefile + per-(rule, universe) `RuleJob` cfg; `render_recipe()` template substitution | +| `engine.job` | `RuleJob` — the typed generator→worker contract | +| `engine.runner` | `run_rule()` — the worker sequence (gates, env check, boundary exec, manifest) | +| `engine.boundary` | `ExecBoundary` protocol, `ExecScope`, `SandboxAttestation`, `get_boundary()` | +| `engine.dask_cluster` | `cluster_for_run()` — the run-scoped LocalCluster | +| `snakemake_executor_plugin_dask` | rules dispatched as dask tasks; SENTINEL-framed output | +| `engine.scratch` | scratch-root resolution, run dirs, the run lock | + +## Engine — sandbox + +| Module | Responsibility | +|---|---| +| `engine.sandbox.policy` | `build_policy()` — the §7 read/write/exec sets, HOME/XDG contract, `EXEC_ALLOWLIST_VERSION` | +| `engine.sandbox._landlock` | vendored ctypes bindings + `abi()` probe | +| `engine.sandbox.wrap` | `wrap_command()`/`wrap_argv()` — ruleset FD + shim argv assembly | +| `engine.sandbox.probe` | capability probe, hermeticity composition, `status_line()` | +| `engine.sandbox.seatbelt` | the generated SBPL profile (macOS) | +| `engine.sandbox.denial` / `hints` | the denial UX: re-stat, classify, two-remedy render, trailer | +| `engine.sandbox.exec_boundary` | `SandboxExecBoundary` — the enforced `ExecBoundary` | + +## Engine — images + +| Module | Responsibility | +|---|---| +| `engine.image.declaration` | `[tool.lightcone.image]` parsing + static refusals; `ImageDeclaration` | +| `engine.image.definition` / `render` | `ImageDefinition` → deterministic Containerfile text (fixed layering) | +| `engine.image.identity` | `compute_tag()` — `lc-env-` | +| `engine.image.builder` / `builder_podman` | `Builder` protocol; the three-file `BuildContext`; podman with pointed error mapping | +| `engine.image.record` | the build record + dpkg snapshot attestation | +| `engine.image.runtime_podman` / `mounts` | the digest-pinned full-stack run wrapper + the mount set | +| `engine.image.machine` | macOS `podman machine` preflight | +| `engine.image` (package) | `ensure_image()`, `resolve_pinned()`, `image_status()` | + +## Engine — readers & export + +| Module | Responsibility | +|---|---| +| `engine.status` | `get_output_status()`, `env_blast_radius()` — offline by invariant | +| `engine.verify` | `verify_outputs()` — tamper/chain checks + provenance notes | +| `engine.tree` | analysis-tree traversal over the resolved ASTRA spec | +| `engine.validation` | post-materialization output shape checks | +| `engine.wrroc` | Workflow Run RO-Crate export | diff --git a/docs/api/io_manager.md b/docs/api/io_manager.md deleted file mode 100644 index ebfe3099..00000000 --- a/docs/api/io_manager.md +++ /dev/null @@ -1,8 +0,0 @@ -# lightcone.engine.io_manager (removed) - -The Dagster IO manager was retired. Output paths are now baked into the -generated Snakefile by [engine/snakefile](snakefile.md), and the canonical -location of every output directory is computed by -[`resolve_output_path`](tree.md) — root outputs land at -`results///`, and path-rooted sub-analyses land at -`/results///`. diff --git a/docs/api/manifest.md b/docs/api/manifest.md deleted file mode 100644 index 345bc451..00000000 --- a/docs/api/manifest.md +++ /dev/null @@ -1,112 +0,0 @@ -# lightcone.engine.manifest - -The integrity layer. Every materialized output gets a sidecar -`.lightcone-manifest.json` written by this module on the host -immediately after the recipe shell exits. - -Source: `src/lightcone/engine/manifest.py`. Schema version: `1` -(`SCHEMA_VERSION = 1` — bump if you change the manifest shape). - -## Public surface - -```python -__all__ = [ - "MANIFEST_FILENAME", # ".lightcone-manifest.json" - "SCHEMA_VERSION", # 1 - "code_version", - "fingerprint_external", - "read_manifest", - "sha256_dir", - "write_manifest", -] -``` - -## `code_version(*, recipe, container_image, decisions) → str` - -Deterministic content hash of everything that defines what the recipe -*does*: the recipe text, the resolved container image identifier, and -the canonicalized decision dict. Returns `"sha256:"`. - -The runtime used to invoke the container (docker / podman / podman-hpc) -is intentionally excluded — the same image produces the same data -regardless of which OCI tool launched it. - -`code_version` is embedded in each rule's `params.cfg` so Snakemake's -`params` rerun-trigger detects drift automatically. - -## `sha256_dir(path) → str` - -Deterministic content hash of a directory tree. Walks recursively, -hashes each file along with its relative path (so renames change the -hash), and excludes: - -- `.lightcone-manifest.json` (chicken-and-egg) -- `.snakemake_timestamp` (touched by Snakemake *after* the rule body - completes — including it would make every hash unstable) - -Raises `FileNotFoundError` if `path` does not exist. - -## `fingerprint_external(path, *, strict=False) → str` - -External (non-manifested) input fingerprint: - -- File: `mtime-size:-` by default; `sha256:` when - `strict=True`. -- Directory: always `sha256:` (via `sha256_dir`). -- Missing path: literal string `"missing"`. - -## `read_manifest(output_dir) → dict | None` - -Read `/.lightcone-manifest.json`. Returns `None` if the -file is missing or unparseable. **Does not** catch `OSError` — a -permission-denied or I/O error is surfaced rather than silently -masquerading as "missing", because it would otherwise hide real -problems from `lc verify` / `lc status`. - -## `write_manifest(*, output_dir, inputs, cfg) → Path` - -Atomically write the manifest for an already-materialized output. -Called from each rule's `run:` block. - -Required keys in `cfg`: - -- `output_id`, `universe_id` -- `recipe`, `container_image`, `decisions` -- `code_version` -- `git_sha`, `lc_version` - -`inputs` is a `dict[str, Path]` mapping declared input id → filesystem -path. For each input, the function reads the upstream manifest if -present and records its `data_version`; otherwise falls back to -`fingerprint_external`. - -Atomicity: writes to `.tmp`, then `os.replace()` rename. -Either both data and manifest exist at the end, or Snakemake reruns -the rule. - -## Manifest shape - -```jsonc -{ - "schema_version": 1, - "output_id": "accuracy", - "universe_id": "baseline", - "code_version": "sha256:…", - "data_version": "sha256:…", - "container_image": "lc-myproject-abc123", - "recipe": "python scripts/eval.py", - "decisions": { "scaling": "standard", "use_pca": "no" }, - "input_versions": { "features": "sha256:…", "labels": "mtime-size:…-…" }, - "git_sha": "...", - "lc_version": "0.4.0", - "host": "saul01", - "slurm_job_id": "1234567", - "finished_at": 1717000000.0 -} -``` - -## Tests - -`tests/test_manifest.py` covers `code_version` determinism, `sha256_dir` -exclusions, `fingerprint_external` modes, and `write_manifest` end-to-end -including the atomic rename. diff --git a/docs/api/runner.md b/docs/api/runner.md deleted file mode 100644 index 8efc1cc6..00000000 --- a/docs/api/runner.md +++ /dev/null @@ -1,14 +0,0 @@ -# lightcone.engine.runner (removed) - -The pluggable runner (`docker`, `venv`, `local`, `slurm`) was replaced by -two thinner pieces: - -- The Snakefile generator at [engine/snakefile](snakefile.md) wraps each - recipe in a ` run --rm ...` invocation at generation time - (or leaves it bare when no container is configured). -- The Dask cluster manager at [engine/dask_cluster](dask_cluster.md) - decides whether the run is local, SLURM-backed via `srun`, or attached - to an external scheduler. - -There is no longer a single "backend" abstraction — those two -modules together cover what the runner used to do. diff --git a/docs/api/site_registry.md b/docs/api/site_registry.md deleted file mode 100644 index 6efca3d7..00000000 --- a/docs/api/site_registry.md +++ /dev/null @@ -1,54 +0,0 @@ -# lightcone.engine.site_registry - -Known-site defaults. When lightcone-cli runs on a recognized site -(NERSC Perlmutter, a lightcone JupyterHub deployment), the matching -entry here supplies site-specific defaults — most importantly the -scratch root and the preferred container runtime. - -Source: `src/lightcone/engine/site_registry.py`. - -## What the module exposes - -- `SITE_DEFAULTS` — a dict mapping site keys (`"perlmutter"`, - `"jupyterhub"`, `"local"`) to a structured defaults dict (display - name, hostname patterns or env markers, backend, container runtime, - `scratch_root`, suggested QoS / constraint / time-limit options). -- `detect_current_site() → HostSite` — the high-level entry point. - Single source of truth for "which site are we on?": environment - markers win over hostname patterns (a pod's hostname is noise; the - injected env is the signal). Returns a falsy `HostSite` when nothing - matches. -- `HostSite` — frozen dataclass bundling the matched site key with its - defaults; `site.get("scratch_root")` etc. -- Lower-level pieces: `detect_site(hostname_or_name)`, - `detect_site_from_env()`, `get_site_defaults(site_key)`, - `list_known_sites()`, `get_site_scratch_deny_rules(site_key)`. - -## Who calls it - -- `lc init` (`lightcone.cli.commands`) — detects the site to surface - the resolved scratch root the run layer will use. -- `lightcone.engine.scratch` — `resolve_scratch_root()` falls back to - the site's declared `scratch_root` (e.g. `$SCRATCH` on Perlmutter, - `$HOME` on a JupyterHub pod) when the project config doesn't pin one. -- `lightcone.engine.container` — `auto` runtime resolution prefers the - site's declared `container_runtime` (`podman-hpc` on Perlmutter, - `kubernetes` on a hub). - -Everything should go through `detect_current_site()` rather than -re-deriving `socket.gethostname() + detect_site + get_site_defaults`. - -## Vestigial pieces - -`get_site_scratch_deny_rules()` and `list_known_sites()` currently -have no callers — they are residue from the removed target system. -The `suggested_options` blocks (QoS/constraint/time-limit guidance) -are likewise declared but not consumed yet. - -## Adding a site - -Append an entry to `SITE_DEFAULTS`. HPC sites match by -`hostname_patterns`; deployment-style sites (pods with arbitrary -hostnames) match by `env_markers`. Declare `scratch_root` for any site -where the default tempdir is wrong — see the `jupyterhub` entry's -comment for why a shared filesystem matters there. diff --git a/docs/api/snakefile.md b/docs/api/snakefile.md deleted file mode 100644 index ed8f3b43..00000000 --- a/docs/api/snakefile.md +++ /dev/null @@ -1,106 +0,0 @@ -# lightcone.engine.snakefile - -Generate `.lightcone/Snakefile` and `.lightcone/snakefile-config.json` -from `astra.yaml`. Both are auto-generated on every `lc run` — never -edit by hand. - -Source: `src/lightcone/engine/snakefile.py`. - -## Public surface - -```python -__all__ = ["generate", "discover_universes", "LIGHTCONE_DIR"] -``` - -## `generate(project_path, *, universes, runtime="none") → (Path, Path)` - -Reads `astra.yaml`, resolves the analysis tree, and writes: - -- `.lightcone/Snakefile` — the workflow. -- `.lightcone/snakefile-config.json` — per-`(rule_key, universe)` config. - -Returns the two paths. - -`runtime` is one of `docker | podman | podman-hpc | none` and is used to -wrap each recipe at generation time (see -[engine.container.wrap_recipe](container.md#wrap_recipe)). Resolution is -done once here, not per rule, so all rules use a consistent runtime. - -## `discover_universes(project_path) → list[str]` - -Sorted list of universe ids from `universes/*.yaml`, or `["default"]` if -the directory is empty / missing. - -## Generated Snakefile shape - -For each output with a `recipe:` block: - -```python -rule : - input: - ="/...", # only for sibling outputs - output: - data=directory(""), - manifest="/.lightcone-manifest.json", - params: - cfg=lambda wc: CFG[""][wc.universe], - run: - shell('printf "▶ [%s]\\n" "{wildcards.universe}" >&2') - shell(params.cfg["shell_command"]) - write_manifest( - output_dir=Path(output.data), - inputs={"": Path(input.), ...}, - cfg=params.cfg, - ) - for _w in validate_output(Path(output.data), params.cfg.get("output_type"), params.cfg["output_id"]): - print(f"\033[33m⚠\033[0m {_w}", file=sys.stderr) -``` - -## `cfg` content - -Per-`(rule_key, universe)` entry written into -`snakefile-config.json`: - -| Key | Source | Used by | -|-----|--------|---------| -| `output_id` | `tree_out.output_id` | `write_manifest` | -| `output_type` | `output_def["type"]` | `validate_output` | -| `universe_id` | universe name | `write_manifest` | -| `recipe` | `recipe.command` | `write_manifest` | -| `shell_command` | `wrap_recipe(recipe, image, runtime)` prefixed with `: lc_code_version=…;` | the rule body | -| `container_image` | raw `container:` spec from astra.yaml | `write_manifest` (for provenance) | -| `decisions` | merged universe decisions | `write_manifest`, `code_version` | -| `code_version` | `code_version(recipe, image_tag, decisions)` | drift detection via Snakemake `params` trigger | -| `git_sha`, `lc_version` | runtime metadata | `write_manifest` | -| `inputs` | resolved input paths (with `{universe}` substituted) | informational | - -## Why we own the container wrap - -Snakemake supports container directives (`container:` and -`--sdm apptainer`), but we deliberately don't use them. Two reasons, -both pragmatic: - -- `--sdm apptainer` adds an extra container layer that defeats - podman-hpc's migrate workflow. -- Default registry resolution on podman fails for - `lc--` tags because they trip - `unqualified-search-registries`. We pass `--pull=never` to skip the - lookup; Snakemake's machinery doesn't make this easy to thread - through. - -## Naming details - -- **`_rule_key(tree_out)`** — `output_id` for root outputs, - `.` for sub-analysis outputs. This is the - user-visible name and the cfg key. -- **`_rule_name(tree_out)`** — same as `_rule_key` but with `.` → - `__` because Snakemake rule names must be Python identifiers. -- **`_output_dir_pattern(tree_out)`** — wildcard path. Root and inline - sub-analyses: `results/{universe}/`. Path-rooted - sub-analyses: `/results/{universe}/`. - -## Tests - -`tests/test_snakefile.py` covers rule generation across root + sub-analyses, -input wiring, container wrapping, `code_version` embedding, and (last -test) parses the generated Snakefile via `snakemake -n`. diff --git a/docs/api/status.md b/docs/api/status.md deleted file mode 100644 index 85f4f31c..00000000 --- a/docs/api/status.md +++ /dev/null @@ -1,63 +0,0 @@ -# lightcone.engine.status - -Manifest-driven status walker. Reads only the per-output -`.lightcone-manifest.json` files; does **not** import Snakemake. - -Source: `src/lightcone/engine/status.py`. - -## Public surface - -### `get_output_status(project_path, *, universe_id) → Iterator[OutputStatus]` - -Yield an `OutputStatus` for every declared output in `project_path`'s -`astra.yaml`, against the named universe. Used by `lc status` and by -external tooling that wants a structured view. - -The function: - -1. Loads and resolves the analysis tree. -2. Loads merged universe decisions (tolerates a missing universe file — - returns an empty dict). -3. For each tree output: - - If it has no `recipe:` → `alias`. - - If no manifest at the output dir → `missing`. - - Otherwise recomputes `code_version` against the current spec and - compares to the manifest's recorded value. Match → `ok`, - mismatch → `stale`. - -### `OutputStatus` (dataclass) - -```python -@dataclass -class OutputStatus: - output_id: str - universe_id: str - analysis_id: str | None # None for root-level outputs - output_dir: Path - status: StatusLiteral # "ok" | "stale" | "missing" | "alias" - manifest: dict | None # None for missing/alias -``` - -### `StatusLiteral` - -```python -StatusLiteral = Literal["ok", "stale", "missing", "alias"] -``` - -## Why `code_version` is the staleness signal - -The manifest records the `code_version` that produced the data. Drift -detection just recomputes the current `code_version` from the live -spec and compares. Anything that touches recipe text, container image -tag, or decisions changes `code_version`; everything else is irrelevant -for staleness. - -For staleness against external inputs (e.g. someone edited a CSV under -`inputs/`), `lc status` doesn't catch it — that's outside `code_version`'s -scope. Use `lc verify` or rely on Snakemake's `mtime`/`input` rerun -triggers in `lc run`. - -## Tests - -`tests/test_status.py` covers the four status branches end-to-end -against tmp projects, including the alias and decision-drift paths. diff --git a/docs/api/targets.md b/docs/api/targets.md deleted file mode 100644 index 4f29d61b..00000000 --- a/docs/api/targets.md +++ /dev/null @@ -1,11 +0,0 @@ -# lightcone.engine.targets (removed) - -The target configuration module is gone. The only remaining global config -is `~/.lightcone/config.yaml`, which today carries one key: - -```yaml -container: - runtime: auto # auto | docker | podman | podman-hpc | none -``` - -It is read by [`lightcone.engine.container.load_runtime`](container.md). diff --git a/docs/api/tree.md b/docs/api/tree.md deleted file mode 100644 index 5e5301b3..00000000 --- a/docs/api/tree.md +++ /dev/null @@ -1,104 +0,0 @@ -# lightcone.engine.tree - -Walk the resolved analysis tree. Used by the Snakefile generator, -`status`, and `verify` to enumerate outputs, resolve `from:` -references, and merge universe decisions across nested sub-analyses. - -Source: `src/lightcone/engine/tree.py`. - -## `TreeOutput` (dataclass) - -```python -@dataclass -class TreeOutput: - output_id: str - output_def: dict # the raw dict from astra.yaml - analysis_id: str | None # None for root-level outputs - analysis_path: str | None # e.g. "./analyses/hod_fitting" - analysis_spec: dict # the sub-analysis spec dict (root spec for root outputs) -``` - -## Public functions - -### `collect_tree_outputs(spec) → list[TreeOutput]` - -Walk the resolved tree (root-level outputs first, then each -sub-analysis under `analyses:`) and return one `TreeOutput` per output -declaration. - -### `collect_tree_inputs(spec) → dict[str, dict]` - -Return `{qualified_id: input_def}` where `qualified_id` is `input_id` -for root inputs and `analysis_id.input_id` for sub-analysis inputs. - -### `resolve_universe_decisions(project_path, spec, universe_id) → dict` - -Load and merge universe decisions from root and sub-analyses. Returns -a flat dict using qualified keys for sub-analysis decisions -(`analysis_id.decision_id`) to avoid collisions. - -Behavior: - -- Loads root universe from `universes/.yaml`. -- For each sub-analysis with `path:`, looks for the corresponding - universe file at `/universes/.yaml`. The - sub-universe id comes from `root_universe.analyses..universe` - if present, otherwise the root universe id. -- `from:` references on sub-analysis decisions (`from: ../parent_decision`) - are resolved to the corresponding root decision value. -- Local sub-decisions not referenced via `from:` are still added. - -### `get_decisions_for_analysis(merged_decisions, analysis_id) → dict` - -Extract the decisions relevant to a specific analysis. Root analysis -(`analysis_id=None`): returns all unqualified keys. Sub-analysis: -returns decisions with the matching `analysis_id.` prefix, stripped to -local names. - -### `resolve_output_path(project_path, tree_output, universe_id) → Path` - -Returns the *parent* directory of the output dir (i.e. the -`results//` directory). The actual output dir is this path -joined with `tree_output.output_id`. - -- Root + inline sub-analyses: `/results//` -- Path-rooted sub-analyses: `//results//` - -### `resolve_container_spec(tree_output, root_spec) → str | None` - -Pick the container declaration in priority order: - -``` -recipe-level > sub-analysis-level > root-level -``` - -Returns the raw spec string (Containerfile path or registry image), or -`None` when no container is declared anywhere. - -### `find_upstream_output(consumer, inp_id, all_outputs) → TreeOutput | None` - -Resolve a recipe input id to the producing `TreeOutput`. Mirrors the -lookup the Snakefile generator does for `rule.input`. Handles: - -- Dotted `.` → match by qualified key. -- Inside a sub-analysis, bare `inp_id` → first try - `.`, then bare. -- `inp_id` referencing an analysis-level input with `from:` pointing - at a sibling output → resolved through that. - -Returns `None` for inputs that refer to external files (no producer -rule). - -### `resolve_input_path(project_path, spec, from_ref, universe_id) → str | None` - -Resolve a `from:` reference on an input to a concrete filesystem path. -Handles: - -- `../parent_input` → root input's `source` (only if absolute). -- `../sibling.output_id` → sibling sub-analysis's results path. -- `sibling.output_id` (no `../`) → same as above (root convenience). - -## Tests - -`tests/test_tree.py` covers all four resolvers, including the `from:` -edge cases for sub-analysis decisions and inputs. diff --git a/docs/api/validation.md b/docs/api/validation.md deleted file mode 100644 index ce020198..00000000 --- a/docs/api/validation.md +++ /dev/null @@ -1,50 +0,0 @@ -# lightcone.engine.validation - -Post-recipe sanity checks. Called by every rule's body after -`write_manifest`. **Never raises** — all problems are returned as -warning strings and printed to stderr. - -Source: `src/lightcone/engine/validation.py`. - -## `validate_output(output_dir, output_type, output_id) → list[str]` - -Inspect the output directory after a successful recipe run. Empty list -means no problems. Returned strings are human-readable and prefixed by -the rule body with a `⚠ ` marker. - -The check fires unconditionally on a few "this is almost certainly -wrong" situations: - -- Output directory does not exist after a successful run. -- Output directory exists but is a file rather than a directory. -- Output directory is empty after a successful run. - -Beyond that, behavior depends on the declared `type:` in `astra.yaml`: - -| `output_type` | Check | -|---------------|-------| -| `metric` | At least one `*.json` file present, parseable, not all-null/all-NaN. | -| `table` | At least one `*.csv` file present; parseable; warns on individual all-NaN numeric columns and on tables where every numeric column is all-NaN. | -| `figure` | At least one `*.png/jpg/jpeg/svg/pdf/eps` file present; warns on zero-byte files. | -| anything else | Empty list (no specific check). | - -## What it does *not* do - -This is a smoke test, not a validator. It does not: - -- Check the schema or shape of metric JSON beyond null-detection. -- Compare against expected values. -- Catch silent computational errors. -- Block the run — warnings are printed but the manifest is still - written. - -For deeper validation, layer your own checks in the recipe (`assert`, -`pydantic`, …). The point of `validate_output` is to flag the cheap -common silent failures: empty directories, NaN-only columns, missing -files. - -## Tests - -`tests/test_validation.py` covers each output-type branch including -the malformed-input edge cases (unparseable JSON, missing CSV, zero-byte -figures, …). diff --git a/docs/api/verify.md b/docs/api/verify.md deleted file mode 100644 index 491f5c7d..00000000 --- a/docs/api/verify.md +++ /dev/null @@ -1,66 +0,0 @@ -# lightcone.engine.verify - -Recompute on-disk hashes and walk the recorded input chain. Catches -tampering, drift, and forged manifests. Like `status`, this module -never imports Snakemake. - -Source: `src/lightcone/engine/verify.py`. - -## Public surface - -### `verify_outputs(project_path, *, universe_id) → Iterator[VerifyResult]` - -Yield a `VerifyResult` for every materialized output (i.e. every output -with a recipe whose directory exists on disk) in the named universe. - -Outputs that aren't materialized at all are silently skipped — that's a -question for `lc status`, not `lc verify`. - -For each materialized output: - -1. Read its manifest. If missing or unparseable → `missing_manifest`. -2. Recompute `sha256_dir(output_dir)`. If it doesn't match the recorded - `data_version` → `tampered_data` (with a `recorded … != actual …` - detail message). -3. Walk recorded `input_versions`: - - For each declared recipe input, look up the upstream output via - `find_upstream_output`. - - If upstream is external (no producer rule) → nothing to chain to. - - If upstream's current manifest is missing → `broken_chain` - ("upstream … missing manifest"). - - If upstream's current `data_version` ≠ recorded → `broken_chain` - ("upstream … data_version drifted"). - - If the input is missing from the manifest entirely → - `broken_chain` ("input … missing from manifest"). -4. Otherwise `passed=True`. - -### `VerifyResult` (dataclass) - -```python -@dataclass -class VerifyResult: - output_id: str - universe_id: str - output_dir: Path - passed: bool - failure: FailureKind | None - detail: str | None = None -``` - -### `FailureKind` - -```python -FailureKind = Literal["missing_manifest", "tampered_data", "broken_chain"] -``` - -## Performance notes - -`sha256_dir` is the dominant cost. Hashing 10 GB of float arrays takes -real wall time. `lc status` is the cheap version that recomputes -`code_version` only — use that for the day-to-day "is this stale?" -question, and `lc verify` for periodic / pre-publication audits. - -## Tests - -`tests/test_verify.py` covers each failure kind end-to-end against tmp -projects, plus the chain-walking through nested sub-analyses. diff --git a/docs/architecture.md b/docs/architecture.md index f8461629..590afa4a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,313 +1,124 @@ # Architecture -The whole story in one sentence: **lightcone-cli is a thin shim over -Snakemake that owns provenance.** This page expands that sentence. - -## Three subsystems - -1. **Snakefile generation** — translate `astra.yaml` into a - `.lightcone/Snakefile` and a sidecar `snakefile-config.json` keyed by - `(rule, universe)`. Snakemake handles the rest of execution. -2. **Manifest layer** — a per-output sidecar JSON written *by us* on the - host immediately after each rule's recipe shell exits. The integrity - contract lives here. -3. **Cluster management** — `lc run` always dispatches through a Dask - scheduler whose lifetime equals the run's lifetime. The cluster - manager picks the right shape (local / SLURM / external) on the fly. - -Everything the user touches is the `lc` CLI on top of these: the engine -(Snakefile generation + cluster management) and the integrity layer -(manifests, `lc status`, `lc verify`). - ---- - -## 1. Snakefile generation - -Generator: [`lightcone.engine.snakefile.generate`](api/snakefile.md). - -For each output in the resolved analysis tree (root + sub-analyses, -expanded by `astra.helpers.resolve_analysis_tree`), the generator emits -one Snakemake rule per output. The rule body is a `run:` block: - -```python -rule : - input: ... # from upstream outputs (sibling rules) - output: - data=directory("results/{universe}/"), - manifest="results/{universe}//.lightcone-manifest.json", - params: - cfg=lambda wc: CFG[""][wc.universe], - run: - shell('printf "▶ [%s]\\n" "{wildcards.universe}" >&2') - shell(params.cfg["shell_command"]) # the recipe (already container-wrapped) - write_manifest(output_dir=Path(output.data), inputs={...}, cfg=params.cfg) - for w in validate_output(...): print(f"⚠ {w}", file=sys.stderr) -``` - -### What goes in `cfg` - -`snakefile-config.json` is keyed by ` → → cfg` where -the inner dict carries: - -- `shell_command` — the recipe pre-wrapped at generation time. When - containers are configured, this looks like - ` run --rm --pull=never -v "$PWD":"$PWD" -w "$PWD" bash -c ''`. - Snakemake's own `container:` directive and `--sdm apptainer` are - intentionally *not* used — we own the runtime end-to-end. -- `code_version` — `sha256(recipe + container_image + decisions)`. - Embedded as a `: lc_code_version=…;` no-op prefix on the shell command - so it lands in any shell trace. -- `recipe`, `container_image`, `decisions`, `output_id`, `output_type`, - `universe_id`, `git_sha`, `lc_version`, resolved input paths. - -### Why pre-wrap, not Snakemake's `container:`? - -Two reasons. First, `--sdm apptainer` adds an extra container layer that -defeats podman-hpc's migrate workflow. Second, registry image resolution -on podman fails for our content-addressed `lc--` tags -because they trip `unqualified-search-registries` in `registries.conf`. -We pass `--pull=never` to skip the lookup entirely; that requires -images to be present locally, which is what `lc build` does. - -### Staleness detection - -The generator does *not* override Snakemake's rerun logic — it just -makes sure drift is visible to it. We default to -`--rerun-triggers code,input,mtime,params`. The `params` trigger is the -one that fires today: `cfg` is per-universe and contains -`code_version`, so any change to recipe / container image / decisions -flows through. - ---- - -## 2. The manifest layer - -Module: [`lightcone.engine.manifest`](api/manifest.md). Filename: -`.lightcone-manifest.json` (constant; `SCHEMA_VERSION = 1`). - -Every successful rule writes a manifest to its output directory. The -write is atomic (`os.replace` rename); a missing or unparseable manifest -re-runs the rule on the next `lc run`. - -### Fields - -```json -{ - "schema_version": 1, - "output_id": "...", - "universe_id": "baseline", - "code_version": "sha256:…", - "data_version": "sha256:…", - "container_image": "lc-myproject-abc123" , - "recipe": "python scripts/compute.py", - "decisions": {...}, - "input_versions": { "": "sha256:…" }, - "git_sha": "...", - "lc_version": "...", - "host": "...", - "slurm_job_id": "...", - "finished_at": 1700000000.0 -} -``` - -### `data_version` exclusions - -`sha256_dir()` skips two filenames: `.lightcone-manifest.json` (chicken -and egg) and `.snakemake_timestamp` (Snakemake touches the directory -*after* the rule body completes — including it would make every hash -unreproducible). - -### `input_versions` semantics - -For each declared recipe input: -- If the input is a sibling output (has its own manifest) → - `data_version` from that manifest. -- Otherwise treated as external → - `mtime-size:-` for files, `sha256_dir(...)` for - directories, `"missing"` for absent paths. - -### What `lc verify` checks - -- **`tampered_data`** — `sha256_dir()` of the on-disk output no longer - matches the recorded `data_version`. -- **`broken_chain`** — a recorded `input_versions[id]` no longer matches - the upstream output's current `data_version`. -- **`missing_manifest`** — the output directory exists but has no - manifest, or the manifest fails to parse. - -### What `lc status` checks - -- **`ok`** — manifest present, recomputed `code_version` matches. -- **`stale`** — manifest present but `code_version` drifted (recipe, - image, or decisions changed). -- **`missing`** — no manifest. -- **`alias`** — output declared without a recipe; materialized only as a - side effect of an upstream. - -`status` reads only manifests. No Snakemake import, no `.snakemake/` -directory required, works on a fresh clone or frozen archive. - ---- - -## 3. Cluster management - -Module: [`lightcone.engine.dask_cluster`](api/dask_cluster.md). - -`cluster_for_run()` is the only entry point. It is a context manager -that yields a Dask scheduler address valid for the duration of the run, -across three branches: - -1. `DASK_SCHEDULER_ADDRESS` already set → yield as-is. We don't own the - cluster, we don't tear it down. -2. `SLURM_JOB_ID` set → start an in-process scheduler bound to the - driver hostname (`SLURMD_NODENAME` or `gethostname()`), then `srun` - one `dask worker` per node across the allocation. Workers advertise - the node's resources via Dask abstract resources (`cpus`, `memory`, - `gpus`). The Snakemake executor plugin maps per-rule - `cpus_per_task` / `mem_mb` / `gpus_per_task` to per-task constraints. -3. Neither → `LocalCluster()` sized to the local machine. - -The scheduler is always in-process so its lifetime equals the run's -lifetime: no service to manage, no orphaned schedulers. - -### The Snakemake executor - -Module: [`snakemake_executor_plugin_dask`](api/dask_executor.md). - -Snakemake calls `run_job(job)`, we translate it to: - -```python -client.submit( - _run_shell, cmd, - resources=_build_resources(job), - pure=False, - key=f"snakejob-{job.name}-{job.jobid}", -) -``` - -The worker shells out to the (already container-wrapped) command. There -is no per-rule "executor logic" to write — recipes are wrapped at -generation time, so the worker just runs them. - ---- - -## Container layer - -Module: [`lightcone.engine.container`](api/container.md). - -Two surfaces: - -- **Build** — `compute_image_tag()` + `build_image()`. Tags are - `lc--` over the Containerfile and dependency - files (`requirements.txt`, `pyproject.toml`, `poetry.lock`, - `Pipfile.lock`, …). Rebuilds happen only when the hash changes. -- **Run-time wrap** — `wrap_recipe()` produces the command string that - the Snakefile generator embeds into each rule. - -Runtime resolution: `~/.lightcone/config.yaml` carries -`container.runtime` (`auto | docker | podman | podman-hpc | none`). -`auto` picks the first usable in `(podman, docker, podman-hpc)`, -skipping docker if its daemon is unreachable. `none` is an explicit -opt-out — recipes run on the host. When `auto` falls back to `none` -silently, `lc run` warns that the manifest's `container_image` field -will misrepresent what actually executed. - -For `podman-hpc`, the build path also runs `podman-hpc migrate ` -so compute nodes can read the image without a registry. - ---- - -## Sub-analysis tree - -`astra.yaml` can declare nested `analyses:` pointing to sub-directories -each with their own `astra.yaml`. The full tree is resolved by -`astra.helpers.resolve_analysis_tree()` before any operation. - -Output paths follow the analysis layout: - -- Root + inline sub-analyses: `results///` -- Path-rooted sub-analyses: `/results///` - -`from:` references on inputs and decisions are resolved by helpers in -[`engine.tree`](api/tree.md). When an output id is ambiguous (the same -name appears in multiple sub-analyses), `lc run` errors and asks for -the qualified `.` form. - ---- - -## Repository at a glance +lightcone-cli is a thin shim over Snakemake plus three layers it owns +substantively: the **environment model** (uv as the only substrate, +identity, the container hatch), the **integrity layer** (per-output +content-addressed manifests), and the **hermeticity layer** (the +sandbox). Everything else — DAG resolution, staleness, parallelism, +retry, locking — is Snakemake's. ```text -src/lightcone/ # PEP 420 namespace package — NO __init__.py -├── cli/ # Click surface -│ ├── __init__.py # exposes main() -│ └── commands.py # init, run, status, verify, build, export -├── engine/ # execution substrate -│ ├── manifest.py # write_manifest, sha256_dir, code_version -│ ├── snakefile.py # generate .lightcone/Snakefile from astra.yaml -│ ├── container.py # docker/podman/podman-hpc build + recipe wrap -│ ├── cloudbuild.py # GCP Cloud Build backend (kubernetes runtime) -│ ├── dask_cluster.py # cluster lifecycle (local/SLURM/Gateway/external) -│ ├── scratch.py # scratch-root resolution, run dirs, run lock -│ ├── status.py # manifest-driven status walker (no Snakemake) -│ ├── verify.py # recompute hashes, walk the chain -│ ├── tree.py # sub-analysis tree helpers -│ ├── validation.py # post-recipe output sanity checks -│ ├── wrroc.py # Workflow Run RO-Crate export -│ └── site_registry.py # known-site defaults (scratch root, runtime) - -src/snakemake_executor_plugin_dask/ # Snakemake executor → dask.distributed - -tests/ # pytest, mirrors src/ -pyproject.toml # hatchling + hatch-vcs; ASTRA + Snakemake as deps + lc (uv tool shim — the launcher) + │ discover → mode-detect → UV_* scrub → converge → delegate + ▼ + /.venv/bin/lc direct mode: the engine from the + │ project's own lock (LC_DELEGATED=1) + │ — or — + podman run … /opt/venv/bin/lc containerized: the whole stack inside + │ the digest-pinned image + ▼ +astra.yaml ── snakefile.generate() ──► .lightcone/Snakefile + snakefile-config.json + │ + snakemake --executor dask (run-scoped LocalCluster) + │ + run_rule() — the worker sequence: + 1. pre-gate env_version(tree) == job's baked env_version + 2. env check uv sync --check / image identity assert + 3. boundary exec sandbox (Landlock/Seatbelt) + offline overlay + 4. post-gate then write_manifest() + │ + results///… + .lightcone-manifest.json ``` -The `lightcone.*` namespace is a PEP 420 implicit namespace package. -**Do not add `src/lightcone/__init__.py`** — that would turn it into a -regular package and break coexistence with future sibling distributions -(`lightcone-ui`, etc.). Any new `lightcone-*` package must live under -`src/lightcone//` and ship only its own subpackage. - ---- - -## Execution flow +## The layers + +### Environment (`engine/environment.py`, `engine/uv_env.py`, `launcher.py`) + +A project is `pyproject.toml` + `uv.lock` + `.python-version`. Mode is +derived: the presence of `[tool.lightcone.image]` (or +`Containerfile.extra`) *is* the escalation into containerized mode. +`env_version` — one length-framed hash over the lock, the interpreter +pin, the closed install-settings list, and the system-layer declaration +— is the environment identity; it sits inside every output's +`code_version`, so environment edits stale exactly what they can +affect. The launcher (`lightcone/launcher.py`) owns the two-hop +delegation: tool env → project-locked engine, with the frozen interface +(argv passthrough + `LC_DELEGATED=1`). + +### Integrity (`engine/manifest.py`, `engine/status.py`, `engine/verify.py`) + +`code_version = sha256({recipe, decisions, env_version, +writable_project})`; `data_version = sha256_dir(output)`. The manifest +(SCHEMA_VERSION 2) is a declared Snakemake output of every rule — a +missing manifest forces a re-run, closing the agent-faked-file +scenario. `status` and `verify` read only manifests and the repo: +offline by invariant. The one shared `code_version()` function is +called by both the generator (write path) and status (read path), so +they can never disagree. + +### Hermeticity (`engine/sandbox/`, `lightcone/_sandbox_exec.py`, `engine/boundary.py`) + +Every recipe and probe executes through the `ExecBoundary`: a +per-job capability probe picks the mechanism (Landlock on Linux — +including inside containers; Seatbelt on macOS), `policy.py` realizes +the declared sets (own-output write, project+inputs read, env + +versioned allowlist + ELF loaders exec, fresh per-recipe HOME/XDG), +and the stdlib-only shim applies the restriction between fork and +exec. The manifest's `hermeticity` field records the *applied* flags — +downgrades are announced, never silent; `--no-sandbox` is recorded as +`{none, open}`. + +### Images (`engine/image/`) + +Modal-inspired internals behind a one-TOML-table user surface: +`declaration.py` parses and statically refuses; `definition.py` + +`render.py` produce a deterministic Containerfile (fixed layering, apt +before sync, offline ENV only in the final stage); `identity.py` +computes `lc-env-` as a pure function of rendered text + +`pyproject.toml` + `uv.lock` (code edits move nothing); +`builder_podman.py` builds with pointed error mapping; +`runtime_podman.py` runs the full stack digest-pinned under +`--net=none`/`--userns=keep-id`; `record.py` keeps the build record +and the dpkg snapshot attestation. `Builder` and the runtime are +protocols — remote builders and other venues return behind them. + +## Repository structure ```text -astra.yaml ── snakefile.generate() ──► .lightcone/Snakefile + .lightcone/snakefile-config.json - │ - snakemake -s … -d … --executor dask - │ - ┌──────────────────────┼──────────────────────┐ - │ │ │ - DAG resolution per-rule run: dask scheduler - (Snakemake) shell(recipe) (LocalCluster / - + write_manifest() SLURM-srun / - external) - │ - └─► results///data - results///.lightcone-manifest.json +src/lightcone/ # PEP 420 namespace — NO __init__.py +├── _sandbox_exec.py # the exec shim (stdlib-only) +├── launcher.py # tool-env launcher / delegation +├── cli/ # Click surface (init, materialize, run, status, verify, build, export) +├── engine/ +│ ├── environment.py # Mode, EnvironmentSpec, env_version, lock scan +│ ├── manifest.py # the integrity layer (SCHEMA_VERSION 2) +│ ├── job.py # RuleJob — the generator→worker contract +│ ├── snakefile.py # Snakefile generator +│ ├── runner.py # run_rule: the worker sequence +│ ├── boundary.py # ExecBoundary seam +│ ├── sandbox/ # Landlock/Seatbelt policy, wrap, probe, denial UX +│ ├── image/ # declaration → render → identity → build → run +│ ├── attestation.py # worker-side runtime capture +│ ├── status.py verify.py # offline readers +│ ├── dask_cluster.py # run-scoped LocalCluster +│ ├── scratch.py tree.py validation.py wrroc.py +│ └── project.py uv_env.py +└── snakemake_executor_plugin_dask/ # rules as dask tasks ``` -What Snakemake owns (we don't write it): DAG construction, topological -execution, parallelism, dry-run, locking, retry, log capture, -per-rule resources, `--rerun-triggers` for staleness detection. - -What we own: a Snakefile generator, the manifest layer (write/read/verify), -a status walker, a verify routine, the Dask cluster manager, the -container-runtime layer, and a Snakemake executor plugin that submits -each rule to a Dask scheduler. - ---- - -## Configuration files - -| File | Scope | Purpose | -|------|-------|---------| -| `astra.yaml` | Project | The spec. Inputs, outputs, recipes, decisions, sub-analyses. | -| `.lightcone/Snakefile` | Project (generated) | Auto-generated by `lc run`. Don't edit. | -| `.lightcone/snakefile-config.json` | Project (generated) | Per-`(rule, universe)` config. | -| `.lightcone/lightcone.yaml` | Project | Tiny scratchpad — currently writes only `target: local`. Not consumed by today's code. | -| `~/.lightcone/config.yaml` | User | `container.runtime`. | - -The `dagster.yaml` and `~/.lightcone/targets/*.yaml` files referenced in -older docs are no longer used — historical residue. +## Key invariants + +- `astra.yaml` carries analysis structure only; the environment lives + in the uv project files. Legacy `container:` keys are ignored. +- The Snakefile and `snakefile-config.json` are regenerated on every + `lc materialize` — never edit them. +- A failing recipe writes no manifest; the `os.replace` in + `write_manifest` is the atomic commit point. +- Recipes are never wrapped at generation time: enforcement happens at + exec time (the boundary), containerization at delegation time (the + launcher). +- Run flags (`--no-sandbox`, `--require-sandbox`) travel to workers via + environment variables, never via cfg — they must not perturb the + content-addressed job identity. +- The manifest records what *actually* ran — mechanism, image digest, + platform — never what documentation says should have run. diff --git a/docs/cli/build.md b/docs/cli/build.md index d53445cf..78e39cbc 100644 --- a/docs/cli/build.md +++ b/docs/cli/build.md @@ -1,90 +1,41 @@ # lc build -Build container images declared in `astra.yaml` (or pre-pull registry -images so `lc run` can use `--pull=never`). - -## Synopsis - -```text -lc build [OPTIONS] -``` - -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--force` | off | Rebuild / re-pull even if the tag already exists locally. | -| `--runtime {docker,podman,podman-hpc,kubernetes}` | resolved from `~/.lightcone/config.yaml` | Override the runtime for this build. | - -## What it does - -For every distinct `container:` value found in the project (root, -sub-analysis, or recipe-level): - -- **Path to a Containerfile** → compute the content-addressed tag - `lc--`, build the image, and (for `podman-hpc`) - migrate it into the per-node container cache. -- **Anything else** (e.g. `python:3.12-slim`, `ghcr.io/foo/bar:tag`) → - pull it into the local image store. This is what lets `lc run` pass - `--pull=never` to the runtime, sidestepping `unqualified-search-registries` - resolution issues with content-addressed tags. - -On the `kubernetes` runtime (a lightcone JupyterHub deployment, where -no local OCI runtime exists) the same command builds through the -deployment's **GCP Cloud Build** service instead: the staged build -context is uploaded to the deployment's build bucket and the resulting -image is pushed as `$LIGHTCONE_REGISTRY/lc-:` — -the same content-addressed identity, so an unchanged environment is a -single registry check and no build at all. Pre-built registry images -are left alone (worker pods pull them directly). Auth is the pod's -Workload Identity; nothing to configure. - -If the runtime is `none` (either by config or because `auto` couldn't -find one), `lc build` prints a friendly note and exits 0. There is -nothing to build. - -## Tag computation - ```text -lc-- -``` - -The hash covers the Containerfile contents plus any of these dependency -files at the project root: - -- `requirements.txt` -- `requirements-dev.txt` -- `requirements-test.txt` -- `pyproject.toml` -- `setup.py` -- `setup.cfg` -- `poetry.lock` -- `Pipfile.lock` - -Edit any one of those and the tag changes. That, in turn, changes -`code_version` in every recipe that uses the image, which marks all -downstream outputs `stale` in `lc status`. - -## Examples - -```bash -lc build # build / pull whatever's missing -lc build --force # rebuild / re-pull everything -lc build --runtime podman-hpc # force the HPC runtime -``` - -## Pre-staging for HPC - -On a login node: - -```bash -$EDITOR ~/.lightcone/config.yaml # container.runtime: podman-hpc -lc build # builds + migrates everything +lc build [--force] ``` -Then submit a SLURM job for `lc run`. The compute nodes will find every -image already cached. - -See [api/container](../api/container.md) for the implementation and -[Architecture](../architecture.md) for why we wrap recipes ourselves -instead of using Snakemake's `container:` directive. +Build the project's environment image (containerized mode). On a +direct-mode project this is an explanatory no-op — there is no image to +build until `[tool.lightcone.image]` is declared. + +## What it builds + +The image is **generated, never authored**: the locked environment plus +the declared system layer render to a Containerfile with a fixed +layering — + +1. the digest-pinned base (the engine's default Debian base, or the + project's declared `base`); +2. the base-contract checks (glibc, `/bin/sh`, apt when + `system-packages` are declared) — each violation is a pointed + build-time refusal, never a raw build log; +3. the apt layer (sorted `system-packages`), **before** the + environment sync, so lock-level system dependencies (sdist builds, + rpy2-style imports) resolve where the system layer actually is; +4. the pinned uv binary and the exact `.python-version` interpreter; +5. `uv sync --locked --exact --no-install-project --compile-bytecode` + into `/opt/venv` — the build context is exactly the rendered + Containerfile, `pyproject.toml`, and `uv.lock`; **project code never + enters an image**; +6. the optional `Containerfile.extra` stage; +7. the final ENV contract (offline overlay — nothing inside a running + image ever touches the network for packages). + +## Identity + +The tag `lc-env-` is a pure function of the repo plus the engine: +code edits never move it, environment edits always do. Builds are +incremental — a tag hit is a no-op; `--force` rebuilds anyway. The +build records the produced image id and a snapshot of the installed +system packages (`.lightcone/image/`, machine-local); execution is +pinned to that record. diff --git a/docs/cli/export.md b/docs/cli/export.md index 49ecc00e..93b92efd 100644 --- a/docs/cli/export.md +++ b/docs/cli/export.md @@ -65,7 +65,7 @@ contains the workflow definition, and a warning is printed: ✓ Wrote WRROC directory: ./wrroc Warning: no materialized outputs were found — the bundle contains only the workflow definition. - This usually means recipes haven't been run yet (try lc run) or the + This usually means recipes haven't been run yet (try lc materialize) or the .lightcone-manifest.json sidecars are missing. Workflow-only bundles will not pass strict Provenance Run Crate validation; that profile requires at least one materialized run. diff --git a/docs/cli/index.md b/docs/cli/index.md index 889ebd36..2d48726c 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -1,39 +1,22 @@ -# CLI Reference - -The `lc` CLI is a thin wrapper around the engine. The user-facing -surface is small on purpose — `astra.yaml` carries the analysis -description, and the CLI is the durable, scriptable way to execute -and audit it. - -## Global behavior - -- `~/.lightcone/config.yaml` is created automatically on first use of - any `lc` command. You do not need to create it manually. -- All commands except `init` walk up from the cwd looking for - `astra.yaml`. If none is found, the command errors out. - -## Commands - -| Command | Purpose | -|---------|---------| -| [`lc init`](init.md) | Scaffold a new ASTRA project (`astra.yaml`, `Containerfile`, `.lightcone/`, MyST report template, optional venv & git). | -| [`lc run`](run.md) | Generate the Snakefile and dispatch through Snakemake + Dask. | -| [`lc build`](build.md) | Build container images declared in `astra.yaml`. | -| [`lc status`](status.md) | Manifest-driven status report. No Snakemake import needed. | -| [`lc verify`](verify.md) | Recompute hashes, walk the input chain, surface tampering. | -| [`lc export`](export.md) | Emit interoperable bundles (Workflow Run RO-Crate) for publication. | - -## Global options - -```text -lc [OPTIONS] COMMAND [ARGS]... - -Options: - --version Show the version and exit. - --help Show this message and exit. -``` - -## Removed commands - -For historical context: `lc dev`, `lc setup`, `lc target`, and `lc update` no -longer exist as explicit commands. See the removal pages for details. +# CLI reference + +Bare `lc ` is the canonical invocation on every machine — no +activation, no `python -m`, no per-venue spelling. The launcher +discovers the project (nearest `astra.yaml` walking up), detects the +mode, converges the environment, and delegates execution verbs to the +project's own locked engine. + +| Verb | Runs in | Does | +|---|---|---| +| [`lc init`](init.md) | tool env | idempotently converge the project scaffold | +| [`lc materialize`](materialize.md) | project engine | execute recipes, write manifests | +| [`lc run`](run.md) | project engine | probe: arbitrary commands in the recipe environment | +| [`lc status`](status.md) | tool env | offline, manifest-driven status report | +| [`lc verify`](verify.md) | tool env | recompute hashes, audit the provenance chain | +| [`lc build`](build.md) | tool env | build the environment image (containerized mode) | +| [`lc export`](export.md) | tool env | publishable RO-Crate bundles | + +"Tool env" verbs work before a lock exists (or offline on a frozen +archive); execution verbs run from the engine version pinned inside the +project's own `uv.lock` — in containerized mode, from inside the +project image. diff --git a/docs/cli/init.md b/docs/cli/init.md index 58feff94..bb0a6907 100644 --- a/docs/cli/init.md +++ b/docs/cli/init.md @@ -1,115 +1,42 @@ # lc init -Converge a directory into an ASTRA project. Idempotent — safe to run -at any time, on an empty directory, a half-scaffolded one, or an -existing project. - -## Synopsis - ```text -lc init [OPTIONS] [DIRECTORY] +lc init [DIRECTORY] [--no-git] [--no-sync] [--check] [--json] [--scratch EXPR] ``` -`DIRECTORY` defaults to `.` (the current directory). - -## Convergence semantics - -Each run creates whatever is missing, repairs the pieces lightcone -manages, and never overwrites files you own: - -- **Created if missing** — every item in the tree below. A directory - that already holds an `astra.yaml` is *adopted*: the spec is left - untouched and only the missing lightcone pieces are added. -- **Repaired** — the managed `.gitignore` block (appended exactly once, - keyed on its `# lightcone-cli` marker), and the stored scratch root - when `--scratch` is passed and differs from the project config. -- **Warned about** — problems `lc` can see but must not fix, reported - in the `warnings` list: an unsupported directory `COPY` in your - Containerfile, an unparseable `.lightcone/lightcone.yaml`. Warnings - don't affect the exit code. -- **Never touched** — anything you authored. - -`--check` reports what a run *would* create or repair, writes nothing, -and exits `1` when the project is not converged. `--json` prints the -report as machine-readable JSON: - -```json -{ - "converged": false, - "created": ["Containerfile"], - "repaired": [".gitignore"], - "unchanged": ["astra.yaml", "..."], - "warnings": [] -} -``` - -Agents driving a project should run `lc init` (or `lc init --check ---json`) at the start of a session to make sure the directory is -workable. +Idempotently converge DIRECTORY (default `.`) into an ASTRA project: +creates whatever is missing, repairs the pieces lightcone manages, and +never overwrites files you own. A directory that already holds an +`astra.yaml` is adopted, not rejected. ## What it creates -The spec scaffold follows the `astra init` boilerplate -(`astra.yaml`, `universes/baseline.yaml`), with the -lightcone-specific pieces layered on top. Inside `DIRECTORY` -(creating it if needed): - -```text -astra.yaml # tiny boilerplate spec with one example output -universes/ - baseline.yaml # the default universe -Containerfile # project image; referenced by `container:` in astra.yaml -requirements.txt # analysis dependencies (numpy, pandas to start) -.gitignore # Python + lightcone state + MyST build output -.lightcone/ - lightcone.yaml # project config: { target: local } (+ scratch_root if --scratch) -results/ - README.md # the materialization contract; outputs land here via `lc run` -myst.yml # MyST report configuration (MySTRA plugin) -index.md # template report referencing astra.yaml elements -.venv/ # Python venv with the analysis dependencies (skipped with --no-venv) -``` - -The boilerplate `container: python:3.12-slim` from the astra -boilerplate is rewritten to `container: Containerfile`, so the project -builds its own content-addressed image and dependencies can evolve -under `lc build`. - -On a known site (NERSC Perlmutter, a lightcone JupyterHub), `lc init` -also prints the detected site and the scratch root that `lc run` will -use for its operational state. +- `astra.yaml` + `universes/baseline.yaml` + `src/` — the ASTRA + boilerplate (analysis structure only; any legacy `container:` line is + stripped — the environment does not live in the spec). +- `pyproject.toml` — a *virtual* uv project (no `[build-system]`) with + `lightcone-cli` as an ordinary locked dependency: the engine lives + inside the experiment's lock. +- `.python-version` — an exact interpreter patch pin. +- `uv.lock` (via `uv lock`) and `.venv` (via + `uv sync --locked --exact --compile-bytecode`; skip with `--no-sync`). +- `AGENTS.md` — the boundary rules for AI agents, appended once to an + existing file. +- `.gitignore`, `.lightcone/` project state, `results/` + README, and a + template MyST report (`myst.yml` + `index.md`). + +## Refusals + +An authored root `Containerfile` is refused with instructions: images +are generated from the lock — delete or rename the file, then re-run. +The file operation is the consent; there is no override flag. ## Options -| Option | Default | Effect | -|--------|---------|--------| -| `--check` | off | Report drift without writing; exit 1 if not converged. | -| `--json` | off | Emit the convergence report as JSON on stdout. | -| `--no-git` | off | Skip `git init`. | -| `--no-venv` | off | Skip venv creation (`uv venv` if available, else `python -m venv`). | -| `--scratch ` | site default | Scratch root for snakemake state, dask spill, and run locks. Shell expressions like `$SCRATCH` are kept verbatim and expanded at run time. | - -> The historical `--target`, `--existing-project`, `--sub-analysis`, -> and `--permissions` flags have been removed. - -## Examples - -```bash -lc init # converge cwd -lc init my-analysis # scaffold/converge ./my-analysis -lc init my-analysis --no-git --no-venv # bare bones -lc init . --scratch '$SCRATCH' # pin the scratch root explicitly -lc init --check --json # is this directory workable? (for scripts/agents) -``` - -## Next steps - -```bash -cd my-analysis -# Describe your analysis in astra.yaml — inputs, outputs, recipes, -# decisions. ASTRA specs are plain YAML; write them by hand or draft -# them with your AI coding assistant of choice. -lc run # materialize the outputs -lc status # check what's ok / stale / missing -myst start # preview the report (requires: npm i -g mystmd) -``` +| Flag | | +|---|---| +| `--check` | report drift without writing anything; exit 1 if unconverged | +| `--json` | machine-readable convergence report | +| `--no-sync` | lock but don't materialize `.venv` | +| `--no-git` | skip `git init` | +| `--scratch EXPR` | pin the scratch root in `.lightcone/lightcone.yaml` (shell expressions kept verbatim) | diff --git a/docs/cli/materialize.md b/docs/cli/materialize.md new file mode 100644 index 00000000..8e18a4de --- /dev/null +++ b/docs/cli/materialize.md @@ -0,0 +1,56 @@ +# lc materialize + +```text +lc materialize [OUTPUTS…] [-u UNIVERSE] [-j N] [-f] [-v] + [--rerun-triggers LIST] [--require-sandbox[=declared-fs]] + [--no-sandbox] +``` + +Materialize the outputs declared in `astra.yaml` — all of them by +default, or the named ones (bare `output_id`, or +`analysis_id.output_id` to disambiguate), across all universes or the +one named with `-u`. + +## What happens + +1. The launcher converges the environment: direct mode syncs `.venv` + from the lock; containerized mode resolves (building if necessary, + with an announcement) the digest-pinned environment image and + re-enters `lc` inside it. +2. If the environment changed since outputs were materialized, the + blast radius is printed up front: `environment changed: N + materialized output(s) are now stale`. +3. A Snakefile is generated from `astra.yaml` (never edit it — it is + regenerated every run) and Snakemake dispatches each rule as a Dask + task on a run-scoped local cluster. +4. Each rule runs the worker sequence: an environment gate (has the + lock changed since the run started?), an environment check + (`uv sync --check` / image identity assert), the recipe inside the + sandbox with the offline overlay, a second gate, and only then the + manifest write. A failing recipe writes no manifest. + +## Sandbox flags + +| Flag | | +|---|---| +| `--require-sandbox` | refuse to run any recipe without an enforcement mechanism | +| `--require-sandbox=declared-fs` | additionally require declared-file scoping | +| `--no-sandbox` | run without enforcement — recorded honestly as `{mechanism: none, fs: open}` | + +## Scheduling flags + +| Flag | | +|---|---| +| `-j / --jobs N` | parallel bound (default: CPU count) | +| `-f / --force` | re-run the named outputs (or everything, when none named) | +| `--rerun-triggers` | Snakemake rerun triggers (default `code,input,mtime,params`) | +| `-v / --verbose` | forward the full executor output | + +## Provenance + +Every produced output directory gains `.lightcone-manifest.json` — +identity (`code_version`, `env_version`, `data_version`), the chain +(`input_versions`), git state, runtime attestation, the image identity +when a container ran, and the `hermeticity` record of the enforcement +that actually applied. Concurrent `lc materialize` invocations on one +project are excluded by a run lock. diff --git a/docs/cli/run.md b/docs/cli/run.md index 3e311be9..9ff5d19c 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -1,97 +1,43 @@ # lc run -Materialize outputs declared in `astra.yaml`. Generates a Snakefile -and dispatches through Snakemake on a Dask cluster. - -## Synopsis - ```text -lc run [OPTIONS] [OUTPUTS]... +lc run [CMD…] [--no-sandbox] [--sandbox-debug] ``` -`OUTPUTS` is zero or more output ids. With no arguments, materializes -everything (Snakemake's `rule all`). - -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--universe`, `-u NAME` | all universes in `universes/*.yaml` (or `["default"]` if none exist) | Restrict to one universe. | -| `--jobs`, `-j N` | `os.cpu_count()` | Parallel jobs / Dask submission concurrency. Passed as both `--cores` and `--jobs` to Snakemake. | -| `--rerun-triggers TRIGGERS` | `code,input,mtime,params` | Comma-separated rerun triggers (forwarded to Snakemake). | -| `--force`, `-f` | off | `--force` when targets are named, `--forceall` otherwise. | -| `--verbose`, `-v` | off | Show the underlying Snakemake / executor chatter and the spawned `snakemake` invocation. | - -## What happens, step by step - -1. Find the project (walk up looking for `astra.yaml`). -2. Discover universes from `universes/*.yaml` (default to `["default"]`). -3. Resolve the container runtime via - `lightcone.engine.container.load_runtime`. If `auto` falls back to - `none` while the spec declares containers, print a loud provenance - warning. -4. Generate `.lightcone/Snakefile` and - `.lightcone/snakefile-config.json` for the selected universes. -5. Translate any explicit `OUTPUTS` into Snakemake target paths - (`/.lightcone-manifest.json`) — this is what tells - Snakemake "build that specific output." -6. Open a Dask cluster context (`local`, `srun`-backed inside - `SLURM_JOB_ID`, or external if `DASK_SCHEDULER_ADDRESS` is set). -7. Spawn `snakemake -s … -d … --cores N --jobs N --executor dask - --rerun-triggers …` with `DASK_SCHEDULER_ADDRESS` in the environment. -8. In the default (non-verbose) path, filter the executor's banner - chatter so the output reads as lightcone's, not Snakemake's. Real - error content always passes through. - -## Output qualification - -When the same `output_id` appears in multiple sub-analyses, you must -qualify it as `.`: +The **probe** verb: run an arbitrary command in byte-for-byte the +recipe environment — the locked interpreter and packages, the sandbox +included. With no CMD, opens a shell there (announced). ```bash -lc run inference # error if 'inference' is ambiguous -lc run hod_fitting.inference # disambiguated +lc run python -c "import astropy; print(astropy.__version__)" +lc run python src/explore.py +lc run # sandboxed shell in the recipe environment ``` -Each rule's body wraps the recipe in a ` run --rm --pull=never --v "$PWD":"$PWD" -w "$PWD" bash -c ''` shell when a -container is configured. After the recipe shell exits, the Snakefile -calls `write_manifest()` host-side and the validation snippet emits -warnings for empty / all-NaN / wrong-extension outputs. +## Semantics -## Examples +- Direct mode ≡ `uv run --locked --exact CMD` from the project root, + inside the sandbox. Containerized mode runs the same command inside + the digest-pinned project image. +- A probe has no output, so its **write scope is the tmp scope only** — + never in-tree. Its read scope is the project plus the union of all + declared inputs. +- `lc run` **never builds an image** — on a containerized project whose + image is missing it errors with the exact `lc build` command, so a + two-second probe can't silently absorb a multi-minute build. -```bash -lc run # all outputs, all universes -lc run --universe baseline # one universe -lc run accuracy # one output -lc run accuracy precision --universe baseline # several -lc run --jobs 4 --verbose # parallel, with stack noise -lc run --force --universe baseline # rebuild everything -lc run --rerun-triggers params,input # tighter staleness -``` +## The rename guard -## Inside SLURM +Outputs are materialized, not run. A first argument naming a declared +output errors before any exec: -```bash -salloc -N 4 ... -lc run --universe baseline -j 16 +```text +outputs are materialized, not run — did you mean: `lc materialize best_fit`? ``` -`lc run` detects `SLURM_JOB_ID`, binds the Dask scheduler to the -driver's hostname, and launches one `dask worker` per node via `srun`. -Workers advertise `cpus`, `memory`, and `gpus` resources. Per-rule -resource hints (`cpus_per_task`, `mem_mb`, `gpus_per_task`) constrain -which workers can pick up which jobs. - -## Provenance gotcha - -If `~/.lightcone/config.yaml` says `runtime: auto` and no runtime is -on PATH, `lc run` falls back to running recipes on the host. Because -each manifest still records the *declared* `container_image`, this is a -provenance lie. `lc run` prints a yellow warning telling you to either -install a runtime or set `container.runtime: none` explicitly. +## Diagnostics -See [api/dask_cluster](../api/dask_cluster.md) for the cluster-shape -decision and [Architecture](../architecture.md) for the full execution -flow. +| Flag | | +|---|---| +| `--sandbox-debug` | open a shell *inside* the sandbox, to see exactly what a recipe can see | +| `--no-sandbox` | run without enforcement (recorded as unsandboxed) | diff --git a/docs/cli/status.md b/docs/cli/status.md index 7985c3a5..e4ced956 100644 --- a/docs/cli/status.md +++ b/docs/cli/status.md @@ -1,67 +1,37 @@ # lc status -Manifest-driven status report for every output declared in -`astra.yaml`. - -## Synopsis - ```text -lc status [OPTIONS] +lc status [-u UNIVERSE] [--json] ``` -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--universe`, `-u NAME` | every universe in `universes/*.yaml` | Restrict to one universe. | -| `--json` | off | Emit machine-readable JSON instead of a styled table. | +Report materialization status for every declared output — **offline and +local-only** by invariant: it reads manifests, `pyproject.toml`, and +the local image record; never the network, never Snakemake state. A +fresh clone of a finished project reports its state with no setup. -## Output +## The header -Per universe, one line per declared output: +Three lines answer the questions nothing else surfaces: +```text +mode: containerized (3 system packages) # or: direct +image: lc-env-9f2c81d44a1b03e7 — built (sha256:…) # or: needs build +sandbox: landlock (fs: declared, network: unenforced) # this host ``` -Universe baseline - ✓ ok accuracy - ✸ stale precision - ✗ miss recall - → alias inference -``` - -Statuses (defined in `lightcone.engine.status.StatusLiteral`): -| Status | Meaning | When you see it | -|--------|---------|-----------------| -| `ok` | Manifest present, recomputed `code_version` matches what the manifest recorded. | The output is up to date. | -| `stale` | Manifest present, but `code_version` drifted. | You changed the recipe, image, or a decision since the last run. `lc run` will re-execute. | -| `missing` | No manifest at the expected output path. | Never built, or the directory was deleted. | -| `alias` | The output has no `recipe:` of its own — it's just a name pointing at a sibling output (typical for ASTRA "promoted" outputs from sub-analyses). | Status is implicitly determined by the upstream. | +## Per-output states -## Why it doesn't import Snakemake - -`lc status` reads only the per-output `.lightcone-manifest.json` files -and recomputes `code_version` against the current spec. It never -imports Snakemake or touches `.snakemake/`. That makes it usable on: - -- A fresh clone before any `lc run`. -- A frozen archive copied off a cluster. -- A read-only workspace. - -If a manifest is missing, the output reports `missing`. If a manifest is -unparseable, `read_manifest` returns `None` and you also see `missing` -— that is the agent-forged-file scenario; investigate with `lc verify`. - -## Examples - -```bash -lc status # every output, every universe -lc status --universe baseline # just baseline -lc status --json # machine-readable JSON output -``` +| State | Meaning | What to do | +|---|---|---| +| `ok` | manifest matches the current `code_version` | nothing | +| `stale` | recipe, decisions, or **environment** drifted since materialization | `lc materialize` re-runs it | +| `missing` | no manifest — never materialized (or produced outside `lc`) | `lc materialize` | +| `pre-v2` | manifest from an earlier schema — not comparable to the current identity | re-materialize when convenient | +| `alias` | a `from:` re-export; no independent state | — | -## Related +After the listing, an environment edit's blast radius is stated +explicitly: `environment changed: N materialized output(s) are now +stale`. -- [`lc verify`](verify.md) — recomputes data hashes too (slower; catches - tampering and broken chains). -- [api/status](../api/status.md) — the Python API. -- [api/manifest](../api/manifest.md) — the manifest schema. +`--json` emits the same information machine-readably (used by CI and +agents). diff --git a/docs/cli/verify.md b/docs/cli/verify.md index 6e1ae216..459d2a22 100644 --- a/docs/cli/verify.md +++ b/docs/cli/verify.md @@ -1,67 +1,30 @@ # lc verify -Recompute hashes for every materialized output and walk the recorded -input chain. Catches tampering, drift, and forged manifests. - -## Synopsis - ```text -lc verify [OPTIONS] -``` - -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--universe`, `-u NAME` | every universe | Restrict to one universe. | - -## Output - -``` -Universe baseline - ✓ ok accuracy - ✗ tampered_data precision recorded 'sha256:abc…' != actual 'sha256:def…' - ✗ broken_chain recall upstream 'features' data_version drifted - ✗ missing_manifest f1 No manifest found at output directory +lc verify [-u UNIVERSE] ``` -Exit code is non-zero if any output failed. +Audit the provenance chain by recomputing hashes — like `lc status`, +offline and Snakemake-free. Exit 1 when any check fails. ## Failure modes -| Failure | What it means | -|---------|----------------| -| `missing_manifest` | The output directory exists but `.lightcone-manifest.json` is missing or unparseable. Most innocent cause: someone deleted the manifest. Most concerning: the directory was created by something other than `lc run`. | -| `tampered_data` | The bytes inside the output directory no longer hash to the `data_version` recorded in the manifest. Files were edited, regenerated outside the harness, or the directory contents differ from what was originally written. | -| `broken_chain` | The manifest records a specific upstream `data_version`, but the upstream's current `data_version` doesn't match. Usually means the upstream was rerun without rebuilding the downstream. Fix: `lc run` the downstream. | - -## Outputs without recipes - -Alias outputs (declared in `astra.yaml` without their own `recipe:`) -are skipped — there's no manifest to verify. They are checked -implicitly via the upstream output they reference. - -## Outputs that aren't materialized - -If an output's directory doesn't exist at all, `lc verify` skips it -(no failure to report). Use [`lc status`](status.md) to know what's -missing in the first place. - -## Examples - -```bash -lc verify # every output, every universe — non-zero exit on any failure -lc verify --universe baseline # just baseline -``` +| Failure | Meaning | +|---|---| +| `tampered_data` | the bytes on disk no longer hash to the recorded `data_version` — the output was edited after materialization | +| `missing_manifest` | an output directory exists with no manifest — it was produced by something other than `lc materialize` | +| `broken_chain` | a recorded upstream `data_version` no longer matches the upstream's current one — the upstream was re-run without rebuilding this output | -## When to run +## Notes -- Before publishing a result. -- After moving a project between machines. -- Periodically on shared archives. -- Whenever `lc status` shows `ok` but the data feels suspicious. +Orthogonal to pass/fail, verify surfaces provenance facts the hashes +can't express, per output: -## Related +| Note | Meaning | +|---|---| +| `unsandboxed` | no enforcement mechanism ran when this output was produced | +| `dirty_tree` | materialized from an uncommitted working tree — the recorded `git_sha` can't fully reproduce it | +| `pre_migration` | an earlier-schema manifest; the hashes it carries are still checked | -- [api/verify](../api/verify.md) — implementation and `VerifyResult`. -- [api/manifest](../api/manifest.md) — the manifest schema and what's hashed. +CI can gate on enforcement with +`lc materialize --require-sandbox=declared-fs`. diff --git a/docs/contributing/backends.md b/docs/contributing/backends.md deleted file mode 100644 index 5c651e2d..00000000 --- a/docs/contributing/backends.md +++ /dev/null @@ -1,42 +0,0 @@ -# Adding an Execution Backend (rewritten) - -The `ASTRAContainerRunner` plugin point is gone. Execution is structured -quite differently now, and "adding a backend" decomposes into one or both -of these: - -## Adding a container runtime - -The supported runtimes are `docker`, `podman`, and `podman-hpc` (plus the -`none` no-op). They are listed in -`src/lightcone/engine/container.py::RUNTIMES`. To add a new one: - -1. Append the binary name to `RUNTIMES` (detection priority is the tuple - order). -2. If detection needs a probe (like the docker-daemon ping), extend - `detect_runtime()`. -3. If `wrap_recipe()` needs different flags for the runtime, branch on - `runtime` there. -4. If post-build migration is required (the `podman-hpc migrate` model), - add a `__migrate(tag)` and call it from `build_image()` / - `pull_image()`. -5. Add tests in `tests/test_container.py`. - -## Adding a Dask cluster shape - -Today the cluster manager has three branches: existing scheduler, SLURM -allocation, local. To add a fourth (for example, a custom GPU farm): - -1. Add a branch to `cluster_for_run()` in - `src/lightcone/engine/dask_cluster.py`. -2. Make sure it advertises the same resource keys (`cpus`, `memory`, - `gpus`) so the [Snakemake executor plugin](../api/dask_executor.md) - can match. -3. Add tests in `tests/test_dask_cluster.py`. - -## Adding a non-Snakemake executor - -In principle Snakemake supports multiple executors and we ship one -(`snakemake_executor_plugin_dask`). If you need a different scheduler, -you can write another Snakemake executor plugin and pass it through -`lc run --executor ` — but that flag does not exist today and would -need to be added to `src/lightcone/cli/commands.py::run`. diff --git a/docs/contributing/hpc-sites.md b/docs/contributing/hpc-sites.md deleted file mode 100644 index 6e30e06f..00000000 --- a/docs/contributing/hpc-sites.md +++ /dev/null @@ -1,24 +0,0 @@ -# Adding an HPC Site - -The old target system is gone; what remains is the lightweight -[`site_registry`](../api/site_registry.md) module, which supplies -per-site defaults (scratch root, preferred container runtime) via -`detect_current_site()`. - -If you want lightcone-cli to behave well on a new cluster, what you -actually need is: - -1. **A container runtime that works on compute nodes.** `podman-hpc` is - the supported case. Wire it up via `~/.lightcone/config.yaml`, or - declare it as the site's `container_runtime` in `SITE_DEFAULTS`. -2. **Dask workers reachable from the scheduler.** `lc run` already does - the right thing inside an `salloc`/`sbatch` allocation — the cluster - manager binds the scheduler to the SLURM canonical hostname and - launches one worker per node via `srun`. See - [api/dask_cluster](../api/dask_cluster.md). -3. **A sane scratch root.** `lc run` keeps its operational state - (snakemake metadata, dask spill, cross-node run locks) under a - scratch root that must honour `flock` — on Perlmutter that means - `$SCRATCH` (Lustre), not DVS-mounted home/CFS. Declare - `scratch_root` in the site's `SITE_DEFAULTS` entry; users can - override it per-project with `lc init --scratch`. diff --git a/docs/design/environment-substrate-evaluation.md b/docs/design/environment-substrate-evaluation.md new file mode 100644 index 00000000..0cd47562 --- /dev/null +++ b/docs/design/environment-substrate-evaluation.md @@ -0,0 +1,340 @@ +# Evaluation: the execution-environment substrate (containers vs pixi vs uv) + +- **Status:** findings report — evidence base for the revised + `execution-environment.md` +- **Date:** 2026-08-14 +- **Method:** 15-agent investigation — 7 parallel researchers (pixi, + pixi-on-HPC/k8s, uv, Snakemake-native deployment, reproducibility + fidelity, prior art, adversarial critique of the draft design), a + 3-judge panel scoring 4 candidate architectures from independent + lenses, a completeness critic, and 4 adversarial fact-checks of + load-bearing claims. Repo claims verified against source at the cited + lines. + +## TL;DR + +**Recommendation: lockfile-first, pixi-preferred.** `pixi.toml` + +`pixi.lock` become the single source of truth for the environment and +the manifest's environment identity (a hash of the lock's resolved +artifacts). The container stops being the environment *definition* and +becomes a derived *transport* of the locked environment, kept only where +a site physically requires an image (GKE pod-is-the-image) or where +performance mandates one (Perlmutter at 100+ nodes). All three judges +picked this shape independently; it is also the pattern the ecosystem +has converged on (Nextflow Wave / nf-core, Snakemake's own pin-file + +`--containerize` guidance, the SciPy/Carpentries pixi-to-Apptainer +teaching standard). + +The single most consequential finding is about the status quo, not the +alternatives: + +> **The current guarantee does not hold.** The image tag hashes *build +> inputs* (Containerfile + requirements.txt — `container.py:368`), while +> the base image is an unpinned `python:3.12-slim` and dependencies are +> installed with unpinned `pip` (`commands.py:507`). The same tag — +> and therefore the same `code_version` — can name different resolved +> environments at different build times. Whatever substrate is chosen, +> the manifest's environment field must become either a resolved image +> digest or a lockfile hash. No system surveyed uses an unpinned +> build-input hash as its environment identity. + +## What the eval evidence actually shows + +The draft design's evidence (CI eval runs 1–4) was re-examined +adversarially. Every observed failure — `ModuleNotFoundError: scipy`, +the host-side `pandas` probe, harness-venv pollution — is a pure +package-presence error. Any single blessed environment reachable through +a run verb (`lc run` backed by a container, pixi, uv, or even a venv) +would have prevented all of them identically. The trace-analysis request +for "exec-in-container" presupposes the current substrate; the +underlying need is *exec-in-the-declared-environment*, which is +substrate-neutral. + +Conclusion: the eval evidence settles "one canonical env + a run verb". +It does **not** settle container-vs-lockfile. That decision must rest on +reproducibility strength, operational fit, and tooling cost — which is +what the rest of this report weighs. + +Likewise, of the draft's three venv rejections, two do not transfer to +lockfile substrates: "not versioned" (a lockfile is versioned, and more +pinned than the current image inputs) and "activation cannot be +plumbed" (run-verb tools need no activation — the draft itself cites +`uv run`/`pixi run` as the dominant prior). Only "wrong environment" +(host Python/BLAS/system libs) discriminates substrates — and +conda-forge pins the interpreter, BLAS, and most system libraries, +leaving only kernel/glibc-floor and non-conda-forge tools as +container-exclusive territory. + +## What each identity mechanism actually pins + +| Mechanism | Pins | Residue (unpinned) | Durability of "same id ⇒ same env" | +|---|---|---|---| +| **Build-input image tag (today)** | nothing durable — hashes the *inputs* to a non-deterministic build | base image drift, pip re-resolution | **fails over time**; the identity claim is false | +| **Digest-pinned OCI image** (base by digest + lockfile inside) | full userland: glibc, system libs, Python, wheels | kernel, hardware, GPU driver | deepest per-build identity, but images are local and registry-less here (`--pull=never`); builds are not reproducible, so laptop/Perlmutter/Cloud Build rebuilds of the same inputs yield *different* digests — a single cross-site identity requires new registry + archival infrastructure | +| **pixi.lock** | every conda artifact by URL + sha256 — including the Python interpreter, BLAS, compiler runtimes, MPI — plus PyPI wheels, per platform | host glibc (conda-forge floor 2.17), kernel; PyPI sdists/editable installs; host leakage (`LD_LIBRARY_PATH`, CUDA driver) | strong: conda-forge has an explicit no-deletion policy; identity is per-platform (same lock hash ⇒ different binaries on macOS vs linux-64 — record platform alongside) | +| **uv.lock** | Python wheels/sdists by sha256, one universal cross-platform file; uv-managed interpreter via `.python-version` (pin an exact patch) | everything outside PyPI: system libs, compilers, MPI, non-Python tools; sdist builds use the host toolchain | durable (PyPI immutable after 72h) but shallow — the largest residue of the pinned options | + +Two important honesty notes that apply to *every* mechanism: + +- None yields bit-for-bit output reproducibility (BLAS kernel dispatch + per CPU, thread scheduling, GPU nondeterminism). The claim lightcone + can make is **pinned environment identity**, never bit-identical + outputs. +- Artifact durability ranks: conda-forge (no-deletion policy) ≈ PyPI + (immutable post-72h) > container registries (retention is policy- + governed; Docker Hub has twice announced deletion policies and twice + reversed them — no contractual guarantee either way) > the current + tag (nothing archived at all). Lockfile identities can re-materialize + from public archives; digest identities need the specific blob to + survive. + +## Substrate deep-dives (key verified facts) + +### pixi + +- `pixi.lock` (format v7) pins per-environment, per-platform: conda + packages by URL + version + build string + sha256/md5; PyPI deps as + wheels + sha256. Checksums re-verified at install. +- One `pixi.toml` with `platforms = ["linux-64", "osx-arm64"]` solves + and locks all platforms from any host — one lock covers mac laptop + + Perlmutter + GKE image. +- **Not pip/uv-installable** — the PyPI `pixi` package is an unrelated + Pixiv downloader. It *is* a single static binary with per-release + `.sha256` files and attestations, so `lc` could bootstrap it with + checksum verification, or it's a fair "one extra easy tool" + (curl/homebrew) under the stated principles. +- `pixi run CMD` needs zero shell integration; `--frozen` executes + exactly the lock. `pixi shell-hook` emits a standalone activation + script usable without the pixi binary (batch jobs, containers). +- conda-forge covers BLAS/LAPACK, HDF5, compilers, and MPI — including + `mpich=*=external_*` builds that bind Cray MPICH on Perlmutter (this + is NERSC's own documented mpi4py recommendation). Caveat from the + critic: pixi-specifically this is *operationally unverified* at NERSC + — it inherits the conda-forge mechanism but no NERSC doc endorses + pixi by name. +- Official container path: `ghcr.io/prefix-dev/pixi` images + a + documented multi-stage pattern (`pixi install --locked` in a build + stage, copy env + `shell-hook` entrypoint into a slim final stage) — + image contents become a function of the lock. `pixi-pack` produces + self-extracting offline archives (wheels only, no sdists). +- Maturity: pre-1.0 (0.76.2, Aug 2026), ~4 releases/month; lock format + at v7 (bumped May 2026), backward- but not forward-compatible. + Governance: VC-funded prefix.dev; the underlying rattler library was + adopted by the conda org in 2024. +- Known operational trap (verified, closed-with-workaround): concurrent + cold `pixi install` from many SLURM tasks on Lustre races + (prefix-dev/pixi#5476). A pixi maintainer confirms `pixi run` is safe + once the env is installed. **Design consequence: `lc` must + pre-materialize the env once (driver side) before any fan-out; workers + only ever run warm.** + +### pixi on HPC — the scale boundary + +A pixi env on disk is a standard conda prefix: multi-GB, tens of +thousands of small files, inheriting every shared-filesystem pathology +NERSC documents for conda envs. NERSC's published benchmarks +(`python-shifter` page): shared-FS Python imports vs containers are at +parity around 10 nodes (~4.9s vs ~4.3s), then diverge superlinearly — +~17s vs ~7s at 100 nodes, ~49s vs ~14s at 500. NERSC "strongly urges" +containers at 100+ nodes; podman-hpc's squashfs migration is the +site-blessed path (demonstrated at 900 nodes). Mitigation for the +direct path at small scale: place envs on `/global/common/software` +(read-only on compute nodes, small-file-optimized) via pixi's `[cache]` +/ detached-environments config. + +**Conclusion: pixi and containers are complements at scale, not +substitutes.** Direct locked-env execution is right up to ~10–20 nodes; +an image rendered *from the same lock* is right beyond that. + +### uv + +- `uv.lock` is a universal cross-platform lockfile (one hash across + OSes); `uv run --frozen` on a fresh clone with only uv installed + reproduces the Python layer deterministically, including the + interpreter (pin an exact patch in `.python-version`). +- `tool.uv.required-environments` can force lock-time failure if any + package lacks a wheel for a named platform — turning the + sdist-hermeticity hole into an upfront error. +- The holes are real for this project's workloads: mpi4py's PyPI wheels + bundle MPICH/OpenMPI-ABI, not Slingshot-optimized Cray MPICH — + performant Perlmutter use requires an sdist build against the host + `cc`, an unlocked toolchain step. CUDA needs per-backend index + plumbing and leaves the host driver unpinned. Non-Python tools have + no representation at all. +- Fact-check correction: Perlmutter moved to SLES 15 SP6 (glibc 2.38) + in Feb 2026, so both manylinux_2_28 and _2_34 wheels install fine — + the glibc drift risk raised during research is moot. +- Verdict: uv.lock is an honest identity for the *Python layer only*. + As the sole substrate it would certify "same environment" the manifest + cannot actually see (BLAS, MPI, toolchains) — for scientific + provenance, worse than an admitted gap. + +### Snakemake-native deployment (can we delete code?) + +Not today, and adopting it would *weaken* the guarantee: + +- Stable `--sdm` offers only conda and apptainer. Conda env caching + keys on the MD5 of the env *YAML* (a spec, not a lock); the pin-file + mechanism (`snakedeploy pin-conda-envs`) **silently falls back to the + unpinned YAML on failure** — best-effort, no better than today's + build-input tags. +- The generic software-deployment plugin interface (`software:` + directive, PR #3339) has been an open draft for ~17 months; pixi + support is a no-PR backlog issue (#3915) that depends on it; the + rootless-container plugin is explicitly WIP against the unmerged + framework. Building on this now contradicts the minimal-custom-tooling + principle more than the existing 5-line recipe wrapper does. +- Fact-check correction: Snakemake's `container:` *does* accept local + SIF paths first-class (no registry needed) — the architecture doc's + registry-based rationale for avoiding `--sdm apptainer` is overstated + and should be reworded (the real residue: podman-hpc-storage tags + aren't directly usable; a `podman save` → SIF conversion would fit). +- Nothing in Snakemake addresses the GKE pod-is-the-image case. +- The env-oblivious wrapper approach (recipes as opaque shell strings) + is fully supported and unaffected — swapping what the wrapper wraps + (`pixi run …` instead of `docker run …`) requires nothing from + Snakemake. Revisit if #3339 merges and a pixi plugin ships. + +### Prior art — where the ecosystem converged + +The convergent pattern across every system surveyed: **a solver-agnostic +env spec is the source of truth, a lockfile pins it, and a container is +a derived per-site rendering.** + +- **Nextflow Wave / nf-core** is the purest expression: per-process + conda specs auto-built into frozen Docker *or* Singularity images + (build templates include `conda/pixi:v1`), with a generated lockfile + retained per container; nf-core migrated ~1300 modules to exactly + this. +- **Snakemake's own guidance**: per-rule conda + pin files, optionally + wrapped in apptainer; `--containerize` derives the Dockerfile from + the conda specs. +- **Metaflow** `@conda`/`@pypi`: resolved per-step envs snapshotted to + an object store, reused identically across local/k8s/Batch — pinned + envs without user-built containers. +- **Dagster/Prefect/DVC** have no per-task env story (image per + deployment, or the user's problem) — no help here. +- **Nix/Bazel hermeticity** is essentially absent from scientific + practice (language overhead, privilege models, conda-ecosystem + incompatibility) — rightly out of scope. +- No surveyed system derives its environment identity from an unpinned + Dockerfile's build inputs. + +## The candidates and the verdict + +All candidates keep the substrate-independent core of the draft design: +the `lc run` → `lc materialize` rename, `lc run CMD` as the environment +runner, the boundary rule, and the manifest chain. + +- **A. Container-canonical (draft + hardening):** digest-pin the base, + lockfile inside the image, keep container.py/cloudbuild.py, laptop + requires docker/podman. +- **B. Pixi-canonical:** pixi everywhere; containers only for k8s. +- **C. uv-canonical:** uv.lock + `uv run` everywhere; containers only + for k8s. +- **D. Hybrid lockfile-first:** the lockfile is the single source of + truth and identity hash; the container is an optional derived + transport (k8s always; Perlmutter at scale), never user-authored. + +Judge scores (custom-code ↓ / install bar / repro strength / agent-agnostic / operational): + +| Candidate | simplicity judge | reproducibility judge | operations judge | winner votes | +|---|---|---|---|---| +| A | 2/2/4/5/3 | 2/2/4/5/3 | 2/2/4/5/3 | 0 | +| B | 4/5/4/5/3 | 4/5/5/5/3 | 4/5/5/5/4 | 0 | +| C | 5/5/3/5/3 | 5/5/3/5/3 | 5/5/3/5/3 | 0 | +| D | 3/5/5/5/5 | 3/5/5/5/5 | 3/5/5/5/5 | **3/3** | + +Why the unanimity holds up under scrutiny: + +- **A** is the only candidate that *adds* custom code while deleting + none, is the worst mac-laptop story (Docker Desktop/colima, arm64 + emulation), and — because images here are local, registry-less, and + non-reproducibly built — its "hardened digest identity" fragments per + site unless new registry/archival infrastructure is added. +- **B** contradicts the HPC research it depends on: NERSC's own numbers + say containers are necessary at 100+ nodes, so B would delete working, + site-blessed podman-hpc machinery only to rebuild it later. +- **C** cannot honestly meet the guarantee for this project's HPC + workloads (Cray MPICH, CUDA, non-Python tools). +- **D** is the only candidate with no bad site, and its residual custom + code (lock → image rendering) is precisely the code every surveyed + system also keeps. + +**Refinement adopted from the simplicity judge and critic: narrow D to +pixi-only** (rather than "pixi preferred, uv accepted"). Dual-lockfile +support was D's main maintenance smell; uv remains how users install +`lc` itself and how dev tooling runs, but the *project environment* has +one definition. + +## Risks and open design questions (from the adversarial passes) + +Carried into the revised design doc; the material ones: + +1. **Invalidation blast radius.** Hashing the whole lockfile into + `code_version` means any dependency addition re-materializes every + output. This is not a regression (the image tag already changes + whenever requirements.txt does), but lock-format bumps (v7 was May + 2026) would add *spurious* invalidation. Mitigation: hash a + normalized projection of the lock (the per-platform sorted list of + artifact URLs + hashes), not the raw bytes — stable across format + churn, still changes exactly when the resolved env changes. +2. **Transport must be recorded.** The same lock hash would label + outputs produced host-side (full filesystem visibility) and + in-container (restricted mount). The manifest should record + `transport` (direct | podman-hpc | kubernetes) and the image digest + when one was used, as attestation fields outside `code_version`. +3. **Orchestration-stack layering.** The worker pod image must carry + lightcone-cli + snakemake + dask; today that's a deliberate separate + layer outside requirements.txt. Under lockfile-first, the derived + image keeps that layer; whether `lc` itself should be + conda-forge-installable (so "just pixi" is fully true) is open. +4. **Escape hatch.** "Never user-authored Containerfile" removes the + exit for deps outside conda-forge/PyPI (TeX, proprietary tools). The + derived image should support a documented extension point (e.g. a + user stage `FROM` the derived env image) rather than none. +5. **Sandbox loss.** Today's container wrap mounts only `$PWD`, + mechanically enforcing path discipline; direct `pixi run` execution + loses that. The boundary rule becomes convention-plus-manifest- + backstop on the direct path — acceptable, but should be stated, not + discovered. +6. **Agent re-lock UX.** An agent adding a dependency mid-analysis must + re-lock (`pixi add` does this atomically); `--frozen` failures need a + pointed error message, and env-change-triggered re-materialization is + the intended behavior, not a bug. +7. **ASTRA coordination.** `container:` is part of the upstream ASTRA + schema; lockfile-first needs a spec conversation (new env + declaration, `container:` becoming derived/optional). Also WRROC + export currently maps the image to a SoftwareApplication — the + lockfile (archived in the crate) becomes the checkable provenance + artifact. +8. **Vendor concentration.** pixi is pre-1.0 from a VC-funded company. + Mitigations: pin the pixi version `lc` bootstraps/requires; the lock + format is open and rattler is conda-org-adopted; degradation path is + the uv-variant of D (with its known holes). +9. **Unverified-at-NERSC.** The no-container MPI story (conda-forge + `mpich external_*` under pixi on Perlmutter) inherits a documented + conda mechanism but has no pixi-specific NERSC endorsement — worth a + 1-day spike before committing the Perlmutter direct path. + +## Sources (primary) + +- pixi lock/run/config/containers: pixi.prefix.dev docs (lock_file, + multi_platform_configuration, cli/run, pixi_configuration, + deployment/container, deployment/pixi_pack), github.com/prefix-dev/pixi-docker, + prefix-dev/pixi#5476, tech.quantco.com/blog/pixi-production +- NERSC: docs.nersc.gov — using-python-perlmutter, nersc-python, + python-shifter (benchmark numbers), containers/podman-hpc/overview, + systems/perlmutter/timeline (SLES 15 SP6, Feb 2026) +- uv: docs.astral.sh/uv — resolution, projects/sync, python-versions, + guides/integration/pytorch, reference/storage +- Snakemake: snakemake.readthedocs.io stable deployment docs; + snakemake/snakemake PR #3339, issues #3915, #2880, #971; + snakemake-software-deployment-plugin-container +- Prior art: docs.seqera.io/nextflow/wave, nf-co.re/blog/2024/ + seqera-containers-part-2, docs.metaflow.org/scaling/dependencies, + iterative/dvc#6115, carpentries-incubator reproducible-ml-workflows, + arXiv:2511.04827 +- Durability: conda-forge.org maintainer docs (no-deletion), + docs.pypi.org (yanking/72h), docker/roadmap#152 + Docker blog + (retention policy reversals) diff --git a/docs/design/execution-environment-rationale.md b/docs/design/execution-environment-rationale.md new file mode 100644 index 00000000..3124917f --- /dev/null +++ b/docs/design/execution-environment-rationale.md @@ -0,0 +1,1186 @@ +# Design rationale: the locked environment is the execution environment + +> **The normative specification is +> [execution-environment.md](execution-environment.md)** — now at +> **v4**, which descoped scale (>~10 nodes) per a requirements change +> and deleted this document's placement tiers, per-project images, and +> at-scale container mode after a 4-lens simplification review. This +> file is the long-form rationale and review record — background, +> alternatives analysis, empirical evidence — **and the documented +> re-add path for the scale-era mechanisms** if large-node-count work +> returns. Where the two disagree, the spec wins. + +- **Status:** rationale & review record, revision 3 (v3.2 — uv-based; supersedes + the pixi-based v2 and the container-canonical v1. Substrate evidence + in [environment-substrate-evaluation.md](environment-substrate-evaluation.md); + adoption evidence in [uv-vs-pixi-adoption.md](uv-vs-pixi-adoption.md). + Revision 2 resolved a 6-agent review (identity/image coherence, + dependency groups, read-only-tier semantics, the worker process + contract, uv flag semantics verified against uv 0.12.3, Ray-vs-dask). + Revision 3 resolves a second 4-agent review: the launcher/placement + bootstrap, uv's PATH-fallback and project-discovery holes + (empirically confirmed), install-selection settings escaping the + identity (empirically confirmed), driver-side code attestation, + interactive-allocation UX, gc liveness, and spec-completeness gaps. + A third verification round (2 agents, both scoring 8/10 with + architectural sign-off) produced the final amendments folded in + below: the mid-run relock gate, the non-syncing exec-direct + delegation hop, the UV_* namespace scrub, install-settings in + `env_key`, and sidecar leases.) +- **Date:** 2026-08-15 +- **Scope:** lightcone-cli CLI surface, `lc init` scaffold, engine + environment handling, the dask execution fabric's environment + contract, derived images, eval prompt +- **Coordination required:** ASTRA spec and the hub deployment + contract — both tracked in Open questions +- **Not in scope:** changes to the Snakemake dask executor's + scheduling semantics; hub deployment charts (only the worker-image + and pod contract they consume); Windows (the scaffold targets + linux-x86_64 and macOS-arm64; other platforms are added by editing + `required-environments`) + +## Summary + +A Lightcone project's reproducibility guarantee — every materialized +output tied to a known, pinned environment — currently rests on a +content-addressed container image whose identity does not actually pin +an environment (it hashes build *inputs* while the base image and pip +resolution float). The v2 draft replaced the substrate with +`pixi.toml`/`pixi.lock`. This revision keeps v2's architecture — +lockfile as the single source of truth and identity, container demoted +to a derived per-venue artifact, `lc run CMD` as the environment +runner — but builds it on **uv**, for the reasons quantified in the +adoption report: uv is the presumptive default tool of the Python +ecosystem (and of every coding agent), the uv-lockfile-first + +derived-container pattern is shipped practice across the modern +execution-infrastructure tier (Flyte/Union ImageSpec, Metaflow +`--environment=uv`, Modal `Image.uv_sync`, ClearML — and Ray's own uv +runtime-env hook, which is this same design implemented inside a +fabric), and this project's workloads — dask over TCP, GPU via PyPI +CUDA wheels — no longer need the conda-forge system layer that was +pixi's decisive advantage. + +Five load-bearing decisions: + +1. **`pyproject.toml` + `uv.lock` + `.python-version` at the project + root are the single source of truth for the execution environment.** + uv is the only tool a user needs on any venue; `lc` itself arrives + through it. The manifest's environment identity (`env_version`) + hashes the declared environment: the lock's resolved artifacts, the + interpreter pin, the install-selection settings, and the declared + system layer. +2. **The engine is inside the experiment's lock.** `lightcone-cli` + (which carries snakemake, dask, and the executor plugin) is a locked + dependency of every project. Driver, SLURM workers, and Gateway + worker pods all run the engine *from the project lock* — version + skew becomes structurally impossible on every lc-managed branch, + and detected-and-fail-fast on the caller-owned external branch. +3. **There is exactly one environment per project.** No dependency- + group splits between prototyping and materialization, between venv + venues and images, or between identity and runtime: what `lc run` + probes is byte-for-byte what recipes execute against, on every + venue. (Per-output environments are a deferred extension with a + stated identity story — see Open questions.) +4. **The container is a cache of the locked environment, never its + definition — and project code never enters an image.** Where a venue + needs an image (Gateway worker pods; optionally podman-hpc at + scale), `lc build` renders the environment into one, tagged by a + hash of the *complete rendered build context*. Code reaches every + venue through the filesystem the venue already shares (working + tree, CFS/Lustre, NFS home). Editing code therefore **never** + triggers an image rebuild; only changing the environment does — + which is exactly when a rebuild is meaningful. +5. **One recipe wrapper on every venue: `uv run --locked --exact`** — + with per-venue *enforcement posture*, not per-venue wrapping. Where + the environment tier is writable (laptop), the wrapper converges a + drifted environment to the lock before running. Where it is shared + or read-only (SLURM common software, worker pods, caller-owned + clusters), workers run with `UV_OFFLINE=1`: a warm environment is a + no-op check, and any drift fails fast instead of thundering-herd- + installing — never silently executing stale, never mutating a + shared or foreign tier mid-run. + +`lc run CMD [ARGS…]` (the environment runner) wraps +`uv run --locked --exact`; `lc materialize` (renamed from `lc run` in +stage 1) keeps its dask fabric — and the fabric stays dask (see "The +execution fabric" for the Ray evaluation). + +## Background + +### What the eval evidence established (unchanged from v2) + +The CI evals' consistent failure mode is environment-boundary +confusion: `ModuleNotFoundError: scipy` on first materialize, host-side +probes failing on packages present only in the recipe env, agents +"fixing" imports by installing into the harness's own venv. Every +failure is a package-presence error; all of them are prevented by **one +canonical environment reachable through a run verb**. That evidence is +substrate-neutral — the substrate choice rests on reproducibility +strength, operational fit, tooling cost, and (newly weighed) adoption. + +### Why the pixi-based v2 draft was revised + +The substrate evaluation chose pixi for one decisive property: a single +`pixi.lock` pins the *system* layer (interpreter, BLAS, MPI, compilers) +that uv cannot see. Three findings since then change the weighing +(details and sources in [uv-vs-pixi-adoption.md](uv-vs-pixi-adoption.md)): + +1. **Adoption is an order of magnitude apart and widening.** uv: + ~196M PyPI downloads/month, a `uv.lock` in 32% of Python repos + created in 2025, native Dependabot/Renovate/PyCharm/CI support, + deep training-data presence in every coding agent. pixi: absent + from every usage survey, ~12 Stack Overflow questions, a 5-person + pre-1.0 vendor. For a tool whose primary interface is an agent + working in a terminal, "the agent already knows uv" is a real + reliability property, not a popularity contest. +2. **The uv-lockfile-first + derived-container pattern is battle-tested + prior art.** Flyte/Union ImageSpec accepts a `uv.lock` and derives a + container whose tag is a deterministic content hash; Metaflow ships + pyproject + uv.lock to Kubernetes workers and re-materializes with + uv; Modal builds images server-side from `uv sync --frozen`; ClearML + agents execute `uv sync --locked`; Ray re-execs every worker + through the driver's own `uv run` flags. The multi-stage uv Docker + pattern and `hash(uv.lock)`-as-cache-key are officially documented, + widely replicated standards. +3. **This project's workloads no longer exercise pixi's advantage — + structurally for MPI, presumptively for the rest.** The execution + fabric is dask over TCP — there is no Cray-MPICH ABI requirement in + the materialization path (structural; verified against the fabric + code). GPU stacks are pinnable from PyPI (NVIDIA's official + `cuda-toolkit` wheels including `nvcc`; pinned PyTorch indexes). + BLAS arrives vendored inside numpy/scipy wheels — identical + binaries from the lock everywhere. The residual pixi-only cases — + BLAS-*variant* control, non-PyPI tools without a container — are + believed absent from current projects but have not been surveyed; + Open questions carries the survey and the explicit pixi-fallback + trigger criteria, so the fallback is a decision rule, not a comfort + clause. + +### What the dask fabric implies + +v2 was written as if recipes execute wherever snakemake runs. They do +not. `lc materialize` starts a run-scoped dask cluster +(`engine/dask_cluster.py` — LocalCluster on a laptop, srun-launched +`dask worker`s inside a SLURM allocation, a created-and-culled Dask +Gateway cluster on the hub) and a custom Snakemake executor +(`snakemake_executor_plugin_dask`) submits each rule as a dask task. On +the worker, `_run_shell` launches a *child snakemake* that executes the +rule's `run:` block. Consequences the environment design must honor: + +- **Workers need the full engine** — lightcone-cli, snakemake, dask, + the executor plugin — plus every recipe dependency. The environment + is one indivisible thing; there is no "orchestration layer" + separable from the project environment in practice (the current + design's separate orchestration image layer is where driver/worker + version skew comes from — the executor's `_unpack_result` exists + solely to tolerate it). +- **Three processes run per rule on a worker**, and each must land in + the locked environment: (S1) the dask worker process itself, (S2) + the child snakemake it spawns, (S3) the recipe subprocess. Snakemake's + `RemoteExecutor` would by default embed the *driver's* + `sys.executable` into the child command; today this is neutralized + by the `--shared-fs-usage` set excluding software-deployment (so + workers invoke plain `python` — see `_build_snakemake_cmd`), and the + design keeps that flag **and** adds a belt-and-braces + `get_python_executable()` override so the property no longer hangs + on a side effect (see the venue table). +- **The Gateway branch is where the rebuild-per-edit pain lives.** + Worker pods run the project image; the current scaffold bakes project + code into the image (`WORKDIR /app`), so editing `src/fit.py` forces + a rebuild-and-push before the next run — even though the executor + already `cd`s into the NFS-mounted project directory that both the + notebook pod and the workers share. The code was always reachable + without the image; the image only ever needed to supply the + *environment*. +- **Every venue already shares a filesystem between driver and + workers**: the working tree (laptop), CFS/Lustre (Perlmutter + allocation), NFS home (hub). "Code travels via the filesystem, env + travels via the lock" is therefore implementable with zero new + transport machinery. + +## Design principles + +- **Honest identity.** The manifest records the *resolved* environment + (artifact URLs + hashes) plus the install-selection settings and the + declared system layer, never the inputs to a non-deterministic + build. The claim is pinned environment identity — not bit-identical + outputs (BLAS kernel dispatch and thread scheduling vary by hardware + under every substrate). Where the guarantee is weaker (host-provided + system packages on venv venues, sdist builds, interpreter builds, + apt-layer contents), the manifest says so in attestation fields + rather than pretending. +- **One source of dependencies.** `pyproject.toml` declares, `uv.lock` + pins; venvs, worker pods, and podman-hpc images are all derived from + it. Nothing else declares packages — no requirements.txt, no + authored Containerfile, no per-venue spec. +- **The engine is part of the experiment.** The lc/snakemake/dask + versions that materialize an output are pinned by the same lock as + the science code's dependencies, and covered by the same + `env_version`. +- **The container is a cache.** Correctness comes from the lock; an + image is a pre-warmed rendering of it for venues where warming from + the filesystem is impossible (pods) or too slow at scale (Lustre, + ~100+ nodes). Project code never enters an image. +- **Harness-agnostic by being boring.** A plain CLI command (`lc run`, + and underneath it `uv run --locked --exact`) behaves identically + under Claude Code, Codex, CI, and a human terminal. No settings + plumbing, env files, hooks, or activation. +- **Opt-in, never imposed.** The locked environment is where *project + code* runs, not a cage around the agent; agents legitimately work on + the host outside it. +- **Stateless.** No long-lived containers or shells; the run-scoped + cluster lifecycle stays exactly as it is. + +## The design + +### Environment definition: `pyproject.toml` + `uv.lock` + `.python-version` + +`lc init` scaffolds, at the project root: + +```toml +# pyproject.toml +[project] +name = "my-analysis" +version = "0.0.0" +requires-python = "==3.12.*" +dependencies = [ + "lightcone-cli==X.Y.Z", # the engine: lc + snakemake + dask + executor + "numpy", + # … science deps accumulate here via `uv add` +] + +[tool.uv] +required-version = ">=0.12" +# lock-time failure when a dependency that has no sdist lacks wheels +# for a target platform (sdist-capable packages can still fall back to +# source builds — see the hermeticity section) +required-environments = [ + "sys_platform == 'linux' and platform_machine == 'x86_64'", + "sys_platform == 'darwin' and platform_machine == 'arm64'", +] +``` + +plus `.python-version` pinning an **exact interpreter patch** (e.g. +`3.12.8`, satisfied by uv-managed python-build-standalone builds on +every venue), and runs `uv lock`. `pyproject.toml`, `uv.lock`, and +`.python-version` are committed. `requirements.txt`, the authored +`Containerfile`, and the venv-bootstrap scaffold are removed; existing +projects are converged by `lc init` (see Migration). + +**One environment, no group splits.** The scaffold declares no +dependency groups, and lightcone's execution path treats the project +environment as indivisible: recipes, `lc run` probes, images, and +`env_version` all cover the same resolved set. Projects may use PEP +735 groups, and uv's *defaults* decide what is installed (the `dev` +group by default) — lightcone does not fight those defaults with +per-context flags, because any identity/image/wrapper disagreement +about groups re-creates the probe-succeeds/materialize-fails failure +mode this design exists to kill. Instead, the **install-selection +settings themselves are part of the identity** (see Environment +identity): flipping `[tool.uv] default-groups` changes `env_version` +even though `uv.lock` is byte-identical — an empirically confirmed +hole in lock-only hashing. Two stated consequences: `uv add --group +dev pytest` re-materializes outputs exactly like any dependency edit +(prefer `uv tool install` for host-side tooling — the lc pattern — +over project dev groups); and packages locked in *non-default* groups +are hashed though never installed, a deliberate over-inclusion that +buys wrapper/identity agreement at the cost of occasional spurious +invalidation (the scaffold's no-groups default makes both cases rare). + +**Virtual by default; packaged is a supported step up.** The scaffold +omits `[build-system]`: uv treats such a project as a *virtual* +project — dependencies are managed and locked, the project itself is +never built or installed. That matches a research repo (recipes invoke +`python src/fit.py`, not `import my_analysis`). A project that wants an +importable package adds a build backend (recommend `uv_build`); uv +then installs it *editable* from the working tree on venv venues — +code content stays outside `env_version` (it is code, attested by +`git_sha`), and images remain code-free because image builds always +pass `--no-install-project`. On **image venues**, packaged-project +support requires the runtime editable install into the pod environment +(a write, with build-backend availability constraints) and is +**deferred in v3** — `lc doctor` flags a packaged project that +declares the `gateway` venue; see Open questions. + +**Venue declaration (informational only).** `[tool.lightcone] +venues = ["local", "perlmutter", "gateway"]` (default `["local"]`) +feeds `lc init --gpu`'s guidance and `lc doctor`'s checks. It is never +part of any identity hash, and execution does not require it — +venue detection at run time stays what the fabric code already does. + +**Bootstrap bar (goal 1).** uv is the single prerequisite on every +venue, and it is the same tool that installs `lc` +(`uv tool install lightcone-cli`). A laptop user needs no Docker, no +conda, no pixi, no second package manager — `uv tool install +lightcone-cli && lc init && lc run python …` is the whole on-ramp. If +`lc` was obtained some other way and uv is absent, `lc` prints the +official one-line installer and stops; it does not bundle bootstrap +machinery. + +### `lc` and the project lock: the launcher contract + +The globally installed `lc` (a `uv tool` shim of the same codebase) is +a **launcher** with five responsibilities, executed in order. The +governing rule: **the launcher owns everything that must happen before +the locked engine can exist** — discovery, placement, hygiene, and the +first convergence — and then hands off by *direct exec*, never through +a second `uv run`. + +1. **Project discovery, lc's way.** Walk up for `astra.yaml` (the + existing `_project_root()` rule). uv's own walk-up discovery is + never trusted: a monorepo-root or vendored `pyproject.toml` can + differ from the lightcone project (empirically confirmed + divergence). If no project is found, project verbs fail with lc's + error — the launcher never lets uv pick a project. +2. **Placement and tier selection.** Resolve the venue from the site + registry (the launcher ships the same registry data as the engine — + it *is* the same package, unpinned), pick the environment tier + (site tier when warm or writable; scratch fallback otherwise — see + Environment placement), and export the placement environment: + `UV_PROJECT_ENVIRONMENT`, `UV_CACHE_DIR`, `UV_PYTHON_INSTALL_DIR`, + `UV_LINK_MODE`. Placement paths are keyed by **`env_key` = + sha256(uv.lock bytes ‖ .python-version bytes ‖ canonical + install-settings JSON)[:16]** — computable by any launcher version + with no lock parsing (the install-settings come from a `tomllib` + read of `[tool.uv]`), so launcher/engine skew cannot misplace an + environment; the placement-path schema is declared stable. + Install-settings are *in* the key because they change the installed + set against a byte-identical lock — without them, two + configurations would share a path and `--exact` syncs would + converge each other's packages away. (`env_key` is a *cache key*, + not the identity — `env_version` remains the manifest's identity + and is computed by the pinned engine only.) +3. **UV_\* hygiene.** Scrub the `UV_*` namespace before anything + touches uv: unset every `UV_*` variable except the placement set it + just exported (and, worker-side, the deliberate overlay — + `UV_OFFLINE`, `UV_PYTHON_DOWNLOADS`). Ambient variables like + `UV_NO_BINARY` or `UV_PYTHON` are an unhashed channel into exactly + the install-selection semantics the identity covers; scrubbed + non-empty overrides are logged. The recipe wrapper's environment + gets the same scrub. +4. **Preflight and first convergence.** Check the placed environment: + lock↔pyproject staleness (an lc-side file check — pointed errors + never depend on uv's message formats), marker consistency, + writability. Where the env is cold or drifted *and the tier is + writable*, the launcher performs the convergence itself — + `uv sync --locked --exact` with captured output — and prints the + removal notice ("removed N packages not in uv.lock — add + dependencies with `uv add`"). Where the tier is read-only and the + env is cold/stale, it fails here with the venue message ("run + `lc env sync` from a login node"), *before* uv can emit a raw + `Permission denied`. Convergence is mechanical uv invocation, safe + in the unpinned launcher; everything semantic (env_version, marker + stamping, manifests) belongs to the engine it is about to exec. +5. **Delegation, per-verb, by direct exec:** + + | Verbs | Where they run | Why | + |---|---|---| + | `materialize`, `run`, `build`, `env sync` — and any verb the launcher does not recognize | **exec `/bin/lc …`** with `LC_DELEGATED=1` as the recursion guard | semantics must match the locked engine. Direct exec (no second `uv run`) means there is no PATH-fallback hole at all: if `/bin/lc` does not exist after a successful sync, the error is precise — "lightcone-cli is not in this project's lock — run `uv add lightcone-cli==X.Y.Z`". Unknown verbs delegate because lock semantics are the safe default (the non-delegating set below is closed) | + | `status`, `verify`, `export`, `env gc` | tool environment, directly | manifest-only or filesystem-janitor work, offline, no sync — a fresh clone's `lc status` must not trigger a multi-gigabyte materialization, and `env gc` must run even when the tier is quota-exhausted and a sync would fail; cross-version manifest compatibility is `SCHEMA_VERSION`'s job | + | `init`, `doctor`, `--version` | tool environment, directly | must work before a lock exists; `lc --version` prints both the launcher version and, inside a project, the locked engine version | + + The delegated engine still self-verifies on startup + (`lightcone.__file__` inside the expected prefix) as + belt-and-braces, and stamps the environment marker (a semantic act + the launcher cannot perform — it requires `env_version`). + +`uv run lc materialize` typed directly still works — the engine it +starts performs the same discovery, detects a placement mismatch +(`sys.prefix` ≠ placed path), and re-execs once through the launcher +path (guarded by `LC_DELEGATED`); the only residue is uv's default +`./.venv` from the outer hop, which `lc env gc` offers to remove on +placement-managed sites. + +`lc init` on a project that already pins an engine never changes the +pin without `--upgrade-engine`. + +This retires the executor's version-skew tolerance +(`_unpack_result`'s bare-int fallback) on lc-managed branches; its +removal is gated on the external-scheduler branch's fingerprint check +landing first, so a skewed caller-owned worker gets a remedy, not a +`TypeError`. + +### Environment identity + +`code_version`'s environment input changes from the resolved image tag +to `env_version = sha256(canonical JSON of the env-input document)`, +mirroring `code_version`'s existing canonical-JSON convention +(`sort_keys`, compact separators): + +```jsonc +{ + "schema": 1, + "python": "3.12.8", // from .python-version + "packages": [ // the lock projection, sorted by (name, version, url) + ["numpy", "2.1.0", "https://…/numpy-…whl", "sha256:…"], + … + ], + "install_settings": { // pyproject knobs that change what gets installed — + "default_groups": [...], // empirically these alter the env while uv.lock is + "no_binary": bool, "no_binary_package": [...], // byte-identical. This is the CLOSED, audited + "no_build": bool, "no_build_package": [...] // set for the supported uv range; widening + }, // required-version obligates a re-audit, and golden + // fixtures cover the per-package variants too + "system_packages": ["texlive-latex-base"],// sorted [tool.lightcone] system-packages + "containerfile_extra": "sha256:…" // or null +} +``` + +The **lock projection** covers every package in `uv.lock` except the +project's own root package, per source type: + +- *registry*: one row per locked artifact — `(name, version, artifact + URL, sha256)`; +- *git*: `(name, version, URL including the resolved commit, null)`; +- *path / directory / editable dependencies*: **refused by default** — + `lc` errors at lock-ingestion time ("path dependencies are outside + the integrity guarantee — publish the package, vendor it, or + override with `[tool.lightcone] allow-path-deps = true`"). Under the + override, the row carries a content hash computed by a **dedicated + tree hasher** — `git ls-files`-based when the path is a git repo, + otherwise an explicit exclude set (`.git`, `.venv`, `__pycache__`) + with symlinks not followed — *not* the results-directory + `sha256_dir()` (which has no ignore rules and would hash a sibling + repo's `.git` churn into the identity); cached per (path, HEAD, + dirty) per run. Path deps are additionally refused when the site + registry declares a read-only env tier (uv revalidates directory + deps at run time — a write). + +The projection is over the lock's *content* — the `version` and +`revision` serialization header fields are excluded by name — so +lock-format churn cannot spuriously invalidate outputs. Golden tests +pin the projection (and the default-groups flip) against lock +fixtures. + +The declared system layer is **inside** `env_version`: an output +produced with `texlive` declared is not the same experiment as one +produced without, even though uv cannot see the difference — this +closes the hole where system-dependency edits would be invisible to +`code_version`. The *ambient* channel to the same install-selection +semantics — `UV_NO_BINARY`, `UV_PYTHON`, and friends exported in a +shell — is closed by the launcher's `UV_*` scrub (responsibility 3): +what the identity hashes is the whole surface that can steer an +install, declared or environmental. + +Two honest scope notes, stated rather than discovered: + +- **`env_version` is platform-independent by choice.** `uv.lock` is + universal; the projection covers all locked platforms, so a re-lock + that changes only another platform's wheels moves `env_version` even + though this venue's binaries are unchanged. The design accepts that + rare spurious invalidation in exchange for one identity across the + laptop→HPC→hub path (a per-platform projection would fragment + identity across exactly the venues we unify); the manifest records + `platform` alongside. +- **The interpreter *build* is attestation, not identity.** The + version pin is hashed; the python-build-standalone build uv + materializes for it is selected by the uv release and differs per + platform. The manifest records `python_build` (the full + `cpython-3.12.8-…` tag) and `uv_version`; two runs differing only in + interpreter build share an `env_version`, in the same residue class + as kernel/glibc. (Image venues do not even carry this residue: the + image pins its interpreter bytes, and the image tag covers the uv + release that chose them.) + +**The invalidation blast radius is real, accepted, and surfaced.** +Because the engine and every group are in the identity, both an engine +upgrade (`uv lock --upgrade-package lightcone-cli`) and a dev-tooling +add re-materialize every output. This is deliberate: the engine writes +the manifests, wraps the recipes, and generates the Snakefile — an +identity that excluded it would certify environments differing in the +one component that touches every output; and group-splitting the +identity was rejected above. The cost is contained by the exact engine +pin (upgrades are deliberate acts), steered by the docs (host tooling +via `uv tool`, not dev groups), and **surfaced at decision time**: +when `lc` detects that the current `env_version` differs from the one +in existing manifests, it prints the scope — "environment changed: +N materialized outputs are now stale". (Projecting the engine's +subtree out of `env_version` into attestation was considered and +rejected: it trades auditability for convenience exactly where +provenance tooling should not.) + +Additional manifest fields, all attestation (outside `code_version`): +`platform`, `python_build`, `uv_version`, `worker_runtime` +(`venv` | `image`), `env_tier` (`project` | `site` | `scratch`), image +tag + digest when an image was used (carried by the existing +`LIGHTCONE_WORKER_IMAGE` plumbing), `sdist_built` (locked packages +that resolved to sdists — their builds ran a host toolchain the lock +cannot see), per-rule `git_sha` + `git_dirty` (see the code-state +contract), and the fingerprint-check strength (`marker-match` | +`versions-only` | `unavailable`). + +### `lc run CMD [ARGS…]` — the environment runner + +``` +lc run python src/fit.py --optimizer nelder_mead --output /tmp/probe +lc run python -c "import scipy; print(scipy.__version__)" +lc run # bare: interactive shell inside the environment +``` + +Semantics: + +- **Equivalent to `uv run --locked --exact CMD`**, implemented via the + launcher contract: preflight + `uv sync --locked --exact` + convergence, then CMD exec'd from the placed environment. `--locked` + semantics: a stale lock fails with a pointed message, never silently + re-resolves. `--exact` is load-bearing: uv's default sync is + additive-only (empirically verified), so without it a stray + `uv pip install` would persist across runs and let a recipe import + packages outside the lock while the manifest stamps a locked + `env_version`. With `--exact`, the environment is converged to + exactly the lock — including *removing* extraneous packages — before + every execution. +- **Convergence is lc-driven, uv-executed** — performed once by the + launcher's preflight step (responsibility 4), as + `uv sync --locked --exact` with captured output. This is what lets + `lc` print its own messages — the removal notice and the + venue-specific errors — instead of parsing uv's unstable stderr: + staleness (`uv.lock` vs `pyproject.toml`) and marker drift are + checked lc-side from the files, so pointed errors do not depend on + uv's message formats across the `>=0.12` range. By the time the + user's command runs, the environment is already exactly the lock. +- **Stateless, exit code propagated, stdio inherited** — identical + under every harness. Bare `lc run` opens `$SHELL` inside the + environment. +- Old-grammar guardrail (from stage 1): when the first argument matches + a declared output id, print `did you mean: lc materialize best_fit?`. + +The boundary rule survives verbatim with one word changed: **"if a +`ModuleNotFoundError` in this command would mean 'fix +`pyproject.toml`', it belongs in `lc run`."** Project-code execution +goes through `lc run`; project tooling and everything else stays on the +host. Agents are taught the rule, not an enforcement mechanism; the +manifest chain remains the backstop. + +Two honest notes carried forward: the old container wrap mounted only +`$PWD`, mechanically enforcing path discipline; direct execution sees +the host filesystem — convention plus the manifest backstop replace +the mount on venv venues. And on the hub, `lc run` executes in the +notebook pod (its system layer) while recipes execute in the derived +image's system layer — same lock, same Python packages, different OS +layer; `worker_runtime` keeps the distinction inspectable, and G4 is +exact at the Python layer, which is where the eval failures lived. + +### Environment placement + +uv's defaults put the venv at `./.venv`, the cache at `~/.cache/uv`, +and managed interpreters at `~/.local/share/uv` — correct on a laptop, +pathological on shared filesystems. The launcher applies placement +from the site registry (see the launcher contract); the values: + +- **Laptop / generic**: defaults; `.venv` in the project (IDE- and + agent-discoverable), self-heal permitted. +- **Perlmutter (site registry)**: environments are + **content-addressed by `env_key`, mirroring images**: + `UV_PROJECT_ENVIRONMENT=/global/common/software///envs/`, + `UV_PYTHON_INSTALL_DIR` beside them, `UV_CACHE_DIR` co-located on + the same filesystem (uv hardlinks cache→env only within one + filesystem), `UV_LINK_MODE=hardlink` stated explicitly. `` + resolves from project config (`.lightcone/lightcone.yaml` + `account:`, seeded by `lc init` from `SBATCH_ACCOUNT` or by + prompting; pointed error when absent); `` is the ASTRA + project name. Keying by `env_key` (lock + interpreter bytes) rather + than `env_version` means identity-only edits (system-package + declarations) do not force a pointless re-sync; concurrent runs with + different locks are disjoint by construction; warm checks are + trivially truthful. `/global/common` is small-file-optimized, + **read-only from compute nodes**, and quota'd (default 10 GB / 1M + inodes, shared per account) — `lc doctor` reports usage, and a + CUDA-class environment may need a quota increase. +- **Perlmutter interactive fallback (scratch tier).** When the driver + is on a compute node (salloc/sbatch) and the site env for the + current `env_key` is absent or stale, failing with "go to a login + node" would strand the interactive workflow (an agent that ran + `uv add scipy` mid-allocation could not proceed at all). The + registry therefore declares a **writable fallback tier**: + `$SCRATCH/.lightcone/envs/`, content-addressed identically; + the engine syncs there, proceeds, records `env_tier: scratch` in + manifests, and prints once: "using scratch-tier environment; run + `lc env sync` from a login node to place it on /global/common". + Same lock, same identity — only the placement (and its at-scale + metadata behavior) differs. +- **Hub notebook pod**: driver venv placed per hub config (off + NFS-home where the deployment provides local scratch); worker pods + use the baked image contract, never a forwarded driver path. + +**Environment lifecycle.** Every convergence the engine performs — +`lc run`, `lc materialize` preflight, `lc env sync`, `lc build` — +finishes by writing/refreshing the marker `lightcone-env.json` +(`env_version`, `env_key`, lc version, `python_build`) into the env it +materialized; marker absence on an otherwise lock-consistent env means +"restamp", never "error" (uv recreates venvs wholesale on interpreter +changes — empirically confirmed — and a user's plain `uv sync` is +legitimate). `lc materialize` and `lc env sync` also touch a **lease +file** (job id + expiry) at run start — in a **writable sidecar**, +`/.lightcone/leases/.json` (gitignored), *not* +inside the env: the site tier is read-only from the compute nodes +where runs live, so an in-env lease could never be written by the +processes that need it, and env mtimes there record last sync, not +last use. `lc env gc` (login-node-only on HPC sites) removes +content-addressed envs beyond `--keep N` (default 3, by sidecar-lease +recency falling back to marker mtime), skipping any env with an +unexpired lease or whose job id is still in `squeue` — closing both +the gc-vs-running-job race and the shared-account race. Interpreters +and cache are uv's to manage (`uv cache prune` documented); `lc env +sync` prints the orphan count with a gc hint. + +### `lc env sync` — environment materialization as an explicit step + +`uv sync --locked --exact` under the placement rules, exposed as its +own verb because on Perlmutter the durable tier is **login-node-only** +(read-only on compute, where the SLURM-branch driver lives). The flow: +sync from a login node once per lock change (`lc env sync` — the +existing `_abort_on_perlmutter_login` guard is updated in stage 2 to +allow exactly this verb, and `lc materialize` on a login node performs +the sync and then refuses cluster start with the submit guidance); +inside allocations the preflight finds the env warm, or falls back to +the scratch tier as above. The preflight — env present and +marker-consistent, else writable, else pointed error — runs for **both +`lc materialize` and `lc run`**, before cluster start / before exec, +so a cold env on a read-only tier surfaces as lc's message, not uv's +raw `Permission denied (os error 13)` (the recipe wrapper additionally +pattern-matches uv's permission/offline errors as a backstop and +prefixes the venue remedy). + +### `lc materialize` and the dask fabric + +Behavior preserved: generate the Snakefile, start the run-scoped +cluster, submit rules as dask tasks, write manifests. Environment +mechanics: + +**The per-venue process contract.** Three processes run per rule on a +worker: (S1) the dask worker, (S2) the child snakemake, (S3) the +recipe. The executor overrides `get_python_executable()` to return +`"python"` — never the driver's `sys.executable` — and the +`--shared-fs-usage` setting that already produces this behavior is +kept and pinned by a test; S2 therefore always resolves from the +worker's `PATH`, which each venue contracts to lead with the locked +environment. S3 is always the uniform wrapper +`uv run --locked --exact --project . -- bash -c ''` — the +explicit `--project .` (cwd is contractually the project root after +the executor's `cd`) keeps the never-trust-uv-walk-up rule +exception-free — resolved under the venue's baked/injected placement, +after `run_rule`'s env_key gate (see the code-state contract). A +generator test asserts the emitted job command contains no absolute +interpreter path and carries the project pin. + +| Branch (`dask_cluster.py`) | S1 (worker) | S2 (child snakemake) | S3 (recipe) env source | Enforcement posture | Code | +|---|---|---|---|---|---| +| LocalCluster (laptop) | project `.venv` | `.venv` via PATH | `.venv` | writable: self-heal (`--exact` converges) | working tree | +| SLURM allocation (srun workers) | content-addressed env (site or scratch tier), inherited from the placed driver | same, via PATH | same, via `UV_PROJECT_ENVIRONMENT` | `UV_OFFLINE=1`, `UV_PYTHON_DOWNLOADS=never` in worker env: warm ⇒ no-op, drift ⇒ loud fail | CFS/Lustre working tree | +| — at-scale mode of the SLURM branch (site-thresholded, optional) | image via `srun podman-hpc run --net host -v "$PWD":"$PWD" -v /global:/global … dask worker` | image PATH (`/opt/venv`) | image env via baked contract | offline (image is complete) | `$PWD` bind-mount | +| Dask Gateway (hub / GKE) | image **is** the pod: `/opt/venv` | image PATH | baked contract | offline (baked `UV_OFFLINE=1`) | NFS home working tree (executor already `cd`s there) | +| Pre-existing scheduler (`DASK_SCHEDULER_ADDRESS`) | caller-owned | caller-owned PATH | caller-owned | executor injects `UV_OFFLINE=1` + `UV_PYTHON_DOWNLOADS=never` into the job env — lc never mutates or cold-installs a caller's environment; fingerprint gate below | caller-owned | + +Notes on the at-scale mode: it exists because NERSC's benchmarks show +shared-filesystem environments losing to squashfs images at ~100+ +nodes. dask workers require reachable advertised addresses, so +`--net host` is mandatory — and NERSC documents `--network=host` as +incompatible with podman-hpc's `--gpu` flag, so the mode is initially +scoped to CPU rules (GPU-at-scale via CDI device injection is a spike +question). It is a launch-wrapping mode of the SLURM branch, not a +fifth branch. + +**Rebuild-per-edit is eliminated by construction.** The image tag is a +pure function of the rendered build context (next section), which +contains no project code. Editing `src/fit.py` changes no input to the +tag; the next `lc materialize` reuses the existing image and picks the +new code up from the shared filesystem. Changing the environment +(deps, system packages, interpreter pin) changes the tag — the one +case where a rebuild is *correct*, and `lc` performs or requests it +explicitly rather than silently. + +**Warm-only workers, enforced rather than assumed.** The driver (or +`lc env sync` on read-only-tier sites) materializes the environment +exactly once before fan-out. Workers on every cluster venue run with +`UV_OFFLINE=1` and `UV_PYTHON_DOWNLOADS=never`: a warm environment +makes the per-rule `uv run --locked --exact` a metadata check plus +exec (no network, no writes — verified against a read-only venv on uv +0.12.3, including git-sourced deps from a warm env), and any drift +fails immediately and loudly instead of racing 100 concurrent installs +onto a shared filesystem. (`--no-sync` was considered and rejected: it +silently disables the `--locked` staleness check — verified — which +would trade the guarantee for the optimization.) The justification is +portability and the shared-FS install herd — *not* missing internet; +Perlmutter compute nodes can reach PyPI, which is exactly why +enforcement matters. On the writable laptop venue, concurrent per-rule +syncs on one `.venv` are serialized by uv's environment lock; the +Perlmutter spike's checklist includes confirming the concurrent +`--exact` behavior empirically. + +**The code-state contract.** Code rides the live shared filesystem, so +a mid-run edit can change what later rules execute. The **driver** +captures code state at each rule dispatch — `git_sha` (cheap HEAD +read) and `git_dirty` (`git status --porcelain`, TTL-cached a few +seconds) — and passes it into the job command for `write_manifest` to +record. Driver-side capture is deliberate: worker pods have no git +binary and would trip git's dubious-ownership check on NFS trees, and +the dispatch-to-execution gap is seconds against a threat model of +mid-run human/agent edits. `lc materialize` warns once at start on a +dirty tree. Dependency edits mid-run fail subsequent rules by design +— but **not** via `--locked`, which passes when an atomic `uv add` +updates lock and pyproject *consistently* mid-run (on writable venues +`--exact` would then silently converge workers to the new lock while +manifests stamp the run-start identity — the exact G3 violation this +design exists to prevent, and venue-asymmetric, since read-only tiers +catch it only by accident). Instead the driver captures **`env_key` at +run start and embeds it in every dispatched job command**; worker-side +`run_rule` re-hashes the shared `uv.lock`/`.python-version`/ +install-settings before invoking the recipe and fails on mismatch with +"lock changed mid-run — re-run lc materialize". Fail-fast, not +prevention: the run-scoped cluster and per-rule attestation bound the +blast radius, and `lc verify` surfaces dirty-tree outputs distinctly. + +**Worker fingerprint check** (extends the existing +`_assert_worker_resources` pattern). At cluster connect, the executor +`client.run`s a probe that reports, per worker: the environment marker +(`lightcone-env.json` in venvs, `LC_ENV_VERSION` ENV in images), the +`importlib.metadata` version of lightcone-cli, and +`os.path.isdir(workdir)` (catching a missing NFS mount before the DAG +starts, not at rule 40). Mismatch fails fast with the venue-specific +remedy. On the **pre-existing-scheduler branch** the marker may not +exist; the check degrades to version-level attestation +(`importlib.metadata` names/versions against the lock projection's +name/version set), and — because this is the one branch where lc does +not own the substrate — a fingerprint below `marker-match` **refuses +to run by default**, with `--allow-unverified-cluster` proceeding and +recording the achieved strength (`marker-match` | `versions-only` | +`unavailable`) in every manifest. The strength name is `marker-match`, +not "verified", deliberately: the marker attests the sync that wrote +it, not the current bytes of every installed file. The check is +point-in-time over currently-connected workers; on Gateway the +homogeneous image makes late joiners equivalent, and a +`SchedulerPlugin` re-check on worker-added events is the hardening +step if the external branch ever needs more. + +Snakemake stays entirely env-oblivious (no `--sdm`, no directives) — +recipes remain opaque shell strings; unchanged from v2, and still +cheaper than the unmerged Snakemake software-deployment framework. + +### Derived images: `lc build` (the environment cache renderer) + +For the Gateway venue (always) and the podman-hpc at-scale mode +(optional), `lc build` renders the environment into an OCI image. +The generated Containerfile follows Astral's documented multi-stage +pattern, with one deliberate deviation: the interpreter comes from +`uv python install` (the same python-build-standalone build family as +the venv venues) rather than a python base image, keeping the +interpreter source uniform across venues. + +- builder stage: digest-pinned base; + `COPY --from=ghcr.io/astral-sh/uv@sha256: /uv /bin/`; + `uv python install` into `/opt/python` + (`UV_PYTHON_INSTALL_DIR=/opt/python`, so `pyvenv.cfg` records an + image-stable interpreter path); `uv sync --locked --exact + --no-install-project --compile-bytecode` into `/opt/venv` + (`UV_PROJECT_ENVIRONMENT=/opt/venv`) from `pyproject.toml` + + `uv.lock` **only** — no project source is ever copied in; +- final stage: slim digest-pinned base + declared system packages + + `/opt/python` + `/opt/venv` + the `uv` binary (worker-side + `uv run --locked` needs it), world-readable (`chmod -R a+rX` — no + uid baking; multi-user hubs map arbitrary uids); +- **baked ENV contract** (the image's, never forwarded from the + driver): `UV_PROJECT_ENVIRONMENT=/opt/venv`, + `UV_PYTHON_INSTALL_DIR=/opt/python`, `UV_PYTHON_DOWNLOADS=never`, + `UV_OFFLINE=1`, `UV_CACHE_DIR=/tmp/uv-cache`, + `LC_ENV_VERSION=`, `PATH=/opt/venv/bin:…`. (The + existing `_worker_environment` forwards `HOME` from the driver for + snakemake's sake; the baked contract makes uv indifferent to it.) + +**Image identity: the tag hashes the complete rendered build +context.** + +``` +tag = lc-env- +``` + +The rendered Containerfile embeds the base-image digests, the uv +binary digest, the interpreter version, the system-package list, the +`Containerfile.extra` stage, and the generator's output shape; the +env-input document (the same canonical JSON `env_version` hashes) +carries the lock projection and the install-selection settings that +steer the builder's `uv sync`. The base and uv digests are **generator +constants shipped with the engine**; since `lc build` always runs the +locked engine, new constants reach a project only through an engine +relock — at which point tag and `env_version` move together, and the +tag *additionally* distinguishes renderings across engine versions +(two projects on different engine pins never share a tag for the same +lock). +This is deliberately a distinct identity from `env_version`: +`env_version` is what the *project declares*; the tag is what a +*specific rendering* contains. The manifest records both. One honest +residue, stated like the sdist one: the apt layer is name-pinned only, +so two builds of one tag at different times can hold different +system-package *versions* — the build records a `dpkg -l` snapshot +into the image (`/opt/lightcone/dpkg-snapshot.txt`, surfaced into the +manifest as attestation) rather than pretending apt is pinnable. + +`lc build` is incremental by construction (tag hit ⇒ no-op); +`lc materialize` on an image venue resolves the tag for the current +build context, triggers the venue's builder if absent — Cloud Build on +the hub, docker/podman locally — or fails with the exact command to +run. **The Cloud Build path is a rework, not a reuse**: the existing +`image_identity`/`_populate_build_context` (which hash Containerfile + +requirements and stage the project tree) are replaced by the new tag +function and a context of exactly three files — rendered +Containerfile, `pyproject.toml`, `uv.lock` — pushed to the same +Artifact Registry with the `lc-env-` ref shape. + +**Declared system packages.** A project needing OS-level libraries +declares them: + +```toml +[tool.lightcone] +system-packages = ["texlive-latex-base"] # apt names; in env_version and the tag +``` + +feeding the generated Containerfile's apt layer, with +`Containerfile.extra` (a stage `FROM` the derived env image, content +hashed into both `env_version` and the tag) as the documented escape +hatch replacing the fully user-authored Containerfile. On venv venues +these packages are host-provided: `lc doctor` reports presence, and +`worker_runtime: venv` in the manifest marks the weaker attestation. +This is the honest edge of the uv substrate — a project for which +host-provided system deps on the direct path are unacceptable is the +pixi fallback's territory. + +**GPU.** CUDA-enabled projects pin their PyTorch index explicitly in +`pyproject.toml` via `[tool.uv.index]` (with `explicit = true`) + +`[tool.uv.sources]` — the only project-mode mechanism (uv's +`--torch-backend` exists solely in the `uv pip` interface; in +particular there is no auto-detection to mis-fire inside image +builds). `lc init --gpu` scaffolds the pinned-index block, choosing +the CUDA level against the **minimum** host driver across the +project's declared venues (a `cu13x` pin can lock out a venue whose +driver trails); `lc doctor` compares the locked CUDA level against +`nvidia-smi` where present, and the manifest records the host driver +as attestation. NVIDIA's `cuda-toolkit` PyPI wheels (including `nvcc`) +cover toolkit needs inside the lock. Multi-node GPU collectives (NCCL +over Slingshot) are out of scope for the dask-over-TCP fabric and are +called out as such. + +**Hermeticity posture.** `required-environments` (scaffolded) makes +locking fail when a package **without an sdist** lacks wheels for a +target venue — an upfront error instead of a Perlmutter surprise. It +does not prevent sdist fallback for packages that have one: uv offers +ban-side controls only (`no-build`, `no-build-package`), no +per-package allowlist, so the design does not pretend to a +"wheels-only with opt-in" setting that cannot be expressed. Instead: +lock ingestion reports any sdist-resolved packages, the manifest +records them (`sdist_built`), and projects wanting hard hermeticity +set `no-build = true` themselves (documented, not scaffolded; the +setting is hashed via `install_settings` either way). sdist builds run +the host toolchain — the residue the substrate evaluation already +named — and the attestation keeps it visible. + +### `lc doctor` — the environment health surface + +Load-bearing in six places above, so specified here rather than +implied: `lc doctor` is read-only, runs in the tool environment, exits +non-zero on hard failures only, and checks — per detected venue — +uv presence/version vs `required-version`; lock/pyproject consistency; +placement-tier usage vs quota; declared system packages present on the +host (venv venues); packaged-project × gateway-venue conflict; +locked CUDA level vs `nvidia-smi`; hub contract items (workdir mount, +registry pull access) where detectable; and sdist/path-dep reports. +Shipped in migration step 4 alongside the placement machinery it +inspects. + +### The execution fabric: dask today, with the seam kept explicit + +The user-level question "could Ray replace dask here?" was researched +as part of this revision (KubeRay, Ray-on-SLURM, the Ray uv +runtime-env hook, dask-gateway's health). Verdict: **keep dask; keep +the fabric-touching surface confined to `cluster_for_run` + the +executor plugin (it already is); put Ray on a re-evaluation clock.** + +- Ray's uv integration (driver launched under `uv run` ⇒ every worker + re-exec'd under the same flags, `working_dir` auto-shipped) is *this + design implemented inside a fabric* — independent confirmation of + the architecture — but it is young: multiple open P1-class issues in + 2025–26 (default hook breaking pip runtime-envs, Python-version + mismatches, argument parsing), the wrong maturity for a + reproducibility product's core this year. +- The one capability Ray uniquely adds — shipping code + env to + workers with **no shared filesystem** (`working_dir` via GCS, + ≤500 MiB, node-cached) — solves a problem this design has already + eliminated by construction on every current venue. It is therefore + the **designated escape hatch** if a shared-FS-less venue ever + appears (superseding the dask `UploadDirectory` note of earlier + drafts). +- Where dask is ecosystem-weakest — dask-gateway is in maintenance + mode (~1 release/year, volunteer-maintained, YARN backend dropped in + 2026.3) — Ray is weakest for lightcone's interaction model: no + Gateway equivalent, Ray Client is officially "for experts only" + (30-second disconnect kill, exact version matching), and the blessed + Ray Jobs API is batch-shaped, inverting the live-driver-in-notebook + flow `lc materialize` uses on the hub. dask/distributed core itself + is healthy (monthly releases, funded maintainers). +- Resource semantics map 1:1 (both schedule on logical, unenforced + cpus/memory/gpus), so a later per-venue swap — e.g. the k8s branch + to KubeRay RayJob if dask-gateway stalls — is a contained change + behind the existing seam, not a rewrite. Re-evaluate when: the Ray + uv hook's P1s close and it is stable-by-default; dask-gateway misses + another annual release; or a no-shared-FS / Ray-native venue becomes + a deployment target. + +Operational risk, stated: the Gateway venue depends on dask-gateway's +continued maintenance and on the deployment exposing the standard +`image` / `environment` / `worker_cores` / `worker_memory` cluster +options (the code already errors helpfully when it doesn't). The +project lock pins dask-gateway (engine-in-lock covers the client +side), and the `DASK_SCHEDULER_ADDRESS` branch is the documented exit. + +**Hub deployment contract** (the pod-level preconditions, now explicit; +consumed by the deployment charts, verified by `lc doctor` on the +hub): worker pods mount the same NFS home at the identical path as the +notebook pod; the pod uid can read the project tree and write +`results/`; nodes hold pull auth for the Cloud-Build target registry +(an unpullable image currently surfaces only as the 600 s zero-worker +timeout — the fingerprint check's workdir probe and `lc doctor` +shorten that loop); the gateway exposes the standard cluster options. + +### What this removes and keeps + +Removed from the user surface: the authored Containerfile, +requirements.txt, the venv-bootstrap scaffold, the build-input image +tag as identity, per-venue recipe wrapping differences, the +driver/worker engine-version skew on lc-managed branches (and +eventually its tolerance shim), and pixi-substrate machinery from v2 +(binary bootstrap, lock-format projection quirks, IDE shims — a uv +`.venv` is natively understood by every IDE). + +Kept, slimmed: `lc build` + Cloud Build (reworked to the code-free +rendered context), the site registry (now also carrying placement +tiers), the entire dask cluster/executor fabric, `podman-hpc migrate` +for the at-scale mode, and the manifest chain. + +New surface, kept deliberately small: `lc env sync` / `lc env gc`, the +launcher contract, the fingerprint probe, `lc doctor`'s check list, +and the `[tool.lightcone]` table (`system-packages`, +`allow-path-deps`, `venues`, `account` via project config). + +## Alternatives considered + +- **pixi-canonical (the v2 draft).** Strictly stronger single-file + pinning (interpreter, BLAS variants, MPI, system libs in one lock) + and the right choice if this project's direct-path workloads needed + conda-forge's system layer. The fabric evidence says they don't + (dask-over-TCP, PyPI CUDA wheels, vendored BLAS); the project survey + in Open questions closes the remaining presumption. The costs are + real: a tool agents don't know, a second package manager next to the + uv that installs `lc`, bespoke bootstrap machinery, IDE shims, an + order-of-magnitude-smaller ecosystem, and no battle-tested + workflow-system precedent. **Retained as the documented fallback** + with explicit triggers (see Open questions): the architecture + (lock → env_version → derived image) is substrate-shaped, so a pixi + variant swaps the definition layer without touching the fabric. +- **Container-canonical (v1), hardened.** Digest-pin the base and lock + inside the image. Still adds code while deleting none, still walls + laptops behind Docker, still fragments identity across sites without + new registry infrastructure — and still forces the rebuild loop this + revision exists to kill. Scored last by all three evaluation judges. +- **uv without the engine-in-lock rule** (lc/snakemake/dask as a + separate tool layer, only science deps locked). Reproduces today's + driver/worker skew and requires shipping a second environment to + workers; the executor's own history (`_unpack_result`) is the + evidence against it. Rejected. +- **Ray as the fabric now.** Evaluated above; rejected for now on + maturity of the uv hook and absence of a Gateway-equivalent hub + story, with defined re-evaluation triggers and a per-venue swap + path. Ray `working_dir` + uv is the designated escape hatch for a + shared-FS-less venue. +- **Runtime code-shipping to workers on dask** (`UploadDirectory` / + wheel upload at connect). Superseded by the Ray escape hatch above; + rejected while the shared-FS invariant holds. +- **Snakemake-native deployment (`--sdm conda/apptainer`).** Unchanged + from v2: keys caching on unpinned YAML, silently falls back from pin + files, no Gateway story, and the plugin framework (PR #3339) remains + an open draft; the pixi/uv plugins that would build on it (issues + #3915, #3251) are unstarted. Revisit if the framework merges. +- **Venv + per-harness activation.** Ruled out in v2 for + activation-plumbing reasons that remain true; `uv run --locked + --exact` is the run-verb resolution of that whole problem class. + +## Migration plan + +Stage 1 (substrate-independent, unchanged from v2 — ship first if not +already landed): `run` → `materialize` rename; `lc run CMD` backed by +the current substrate; output-id hint; docs/eval-prompt grammar; tests. + +**Stage 2 — uv substrate.** Named constants used below: recursion +guard `LC_DELEGATED=1`; convergence-consent flag +`--accept-containerfile-loss`; `lc verify` pre-migration state +`pre_migration`; the Containerfile diff baseline is "any historical +scaffold template modulo the version-pin line". + +1. `lc init`: scaffold `pyproject.toml` (engine dep, required-version, + required-environments) + `.python-version`; `uv lock`; converge + existing projects — requirements.txt → dependencies mechanically; + the authored Containerfile is **refuse-and-report**: if it deviates + from the baseline beyond the pip-install lines, `lc init` emits the + diff plus a checklist ("these RUN apt-get lines need + `[tool.lightcone] system-packages` entries or Containerfile.extra") + and requires `--accept-containerfile-loss` — never a silent drop of + a system layer. `lc init` stops writing `container:` into + astra.yaml; the emitted spec is validated against the current ASTRA + release. +2. Launcher contract: astra.yaml-first discovery; placement + tier + selection (env_key incl. install-settings); `UV_*` scrub; + preflight + first convergence with captured output; per-verb + direct-exec delegation (`/bin/lc`, `LC_DELEGATED`); engine + self-verification; unknown-verb delegation; dual `--version`. +3. Engine: `env_version` (the canonical env-input document — golden + tests including the default-groups flip and a path-dep fixture) + replaces the image tag inside `code_version`; manifest gains the + attestation fields (`platform`, `python_build`, `uv_version`, + `worker_runtime`, `env_tier`, image tag/digest, `sdist_built`, + per-rule `git_sha`/`git_dirty`, fingerprint strength); + `SCHEMA_VERSION` bump; `lc verify` reports `pre_migration` + manifests distinctly. During the ASTRA window the Snakefile + generator ignores `container:` with a deprecation warning; + `wrap_recipe`/`resolve_container_spec` are replaced by the uniform + uv wrapper. +4. Fabric + placement: recipe wrapper → `uv run --locked --exact`; + `get_python_executable()` override + `--shared-fs-usage` pinned by + test; generator test for absolute interpreter paths; worker env + overlay `UV_OFFLINE=1` / `UV_PYTHON_DOWNLOADS=never` on all cluster + venues including the external branch; `lc env sync`/`lc env gc` + with leases; scratch-tier fallback; `_abort_on_perlmutter_login` + updated per the env-sync section; site-registry placement values + (Perlmutter tiers, `account` resolution); fingerprint probe + + marker lifecycle; external-branch refusal gate + (`--allow-unverified-cluster`); driver-side code-state capture; + run-start env_key capture + worker-side `run_rule` gate; + `lc doctor`; sweep operator-facing error strings for pre-uv + remediation advice (`pip install distributed`, sbatch-activation + wording). +5. `lc build`: Containerfile generator (code-free, + `--no-install-project`, baked ENV contract, dpkg snapshot); + rendered-context tag; `[tool.lightcone] system-packages`; + `Containerfile.extra`; Cloud Build rework (new three-file context + stager, `lc-env-` refs, `image_identity`/`compute_image_tag` + replaced); Gateway branch resolves/builds by tag. +6. ASTRA coordination: environment declaration = project + pyproject/uv.lock; `container:` derived/optional; WRROC archives + `uv.lock` as the checkable environment artifact. (Steps 1/3 are + written to be safe in the interim window regardless of ASTRA's + timeline.) +7. Eval: re-baseline with the new grammar; the eval doubles as the + live test that the boundary rule lands better when `lc run` + requires nothing but uv. +8. Cleanup (post-window, gated on the external-branch fingerprint + check): drop `_unpack_result`'s bare-int tolerance; mark pixi-era + design docs superseded. + +Test obligations per step, mirroring the repo's test patterns: golden +projection fixtures (step 3); launcher discovery/delegation/self-check +unit tests (step 2); env-sync preflight + lease/gc tests against a tmp +tree (step 4); generated-Containerfile snapshot + tag determinism +tests and a `snakemake -n` parse test (steps 3/5); CliRunner coverage +for the new verbs (steps 1/4). + +No backward compatibility (pre-1.0 stance, unchanged): existing +projects converge via `lc init`, existing manifests report as +`pre_migration` under `lc verify`. + +**The Perlmutter spike (gates stage-2 step 4 defaults).** One day on +Perlmutter before freezing the SLURM-branch defaults: (a) verify a +no-op `uv run --locked --exact --offline` against the read-only +content-addressed env performs zero writes; (b) measure first-rule and +steady-state wrapper latency at ~32 nodes on `/global/common`; (c) +smoke-test the podman-hpc at-scale mode with `--net host` (and probe +CDI GPU injection); (d) confirm interpreter + venv + cache co-location +and quota headroom for a CUDA-class lock; (e) exercise the +salloc-interactive scratch-tier fallback end-to-end; (f) confirm +concurrent `--exact` syncs serialize cleanly on one env. + +## Open questions + +1. **Site threshold for the podman-hpc at-scale mode** — node count at + which SLURM workers switch from shared venv to containers + (evaluation data suggests ~10–20 nodes as the crossover region); + per-site override in the registry; gated on the spike. +2. **Project survey / pixi-fallback triggers** — inventory current + lightcone projects for conda-forge-only needs (BLAS-variant + control, non-PyPI tools needed on the *direct* path). Triggers that + would activate the pixi variant of this design: a real project + blocked by either need, or the CUDA-on-PyPI path regressing. + Absent a trigger, the presumption in Background stands. +3. **Packaged projects on image venues** — runtime editable install + into the pod env (writable `/opt/venv`, offline build backend). + Deferred; `lc doctor` flags the combination. +4. **Per-output environments** — PEP 735 groups map naturally + (`lc run --group heavy`, per-group projection hashes); deferred + until a real project needs it; the one-environment rule is exactly + what this extension would relax deliberately rather than + accidentally. +5. **ASTRA spec evolution** — the `container:` field's + derived/optional future and whether the spec should reference the + environment declaration explicitly; WRROC archiving of `uv.lock`. + (Cross-repo dependency; migration steps are written to be safe in + the interim.) +6. **`lc` on alternative install channels** — is `uv tool install` + plus pip enough, or is a conda-forge package worth maintaining for + hub base images? +7. **Wheel variants** — PEP 825 (with Astral as co-author) would let + uv pin hardware-specific binaries natively; re-check early 2027 and + fold into the GPU guidance when it ships. +8. **Warm-start ergonomics** — `uv run --locked --exact` on a warm env + is milliseconds on local disk; the spike measures the shared-FS + figure. If agent loops ever need less, a long-lived shell is a + contained future optimization, explicitly not built now. +9. **Fabric re-evaluation clock** — revisit Ray per the triggers in + "The execution fabric" (hook P1s closed / dask-gateway stalls / + shared-FS-less venue). + +## Evidence appendix + +- CI eval history (PR #168): runs 1–4 — every remaining failure is + environment-boundary confusion; evidence for one canonical env + behind a run verb, substrate-neutral. Detail in + [environment-substrate-evaluation.md](environment-substrate-evaluation.md). +- Substrate comparison and judge scoring (lockfile-first unanimous), + identity-mechanism table, durability ranking: + [environment-substrate-evaluation.md](environment-substrate-evaluation.md). +- Adoption quantification (uv vs pixi), shipped uv-lockfile-first + systems (Flyte/Union, Metaflow, Modal, ClearML), canonical uv Docker + pattern, CUDA-on-PyPI status, PEP 817/825 timeline: + [uv-vs-pixi-adoption.md](uv-vs-pixi-adoption.md). +- Fabric ground truth: `src/lightcone/engine/dask_cluster.py` (four + cluster branches, run-scoped lifecycle, `LIGHTCONE_WORKER_IMAGE` + attestation, `_worker_environment` HOME forwarding), + `src/snakemake_executor_plugin_dask/executor.py` + (child-snakemake-on-worker model, `cd workdir_init`, + `_unpack_result`), `src/lightcone/cli/commands.py` + (`_build_snakemake_cmd` shared-fs-usage mitigation, + `_abort_on_perlmutter_login`), `src/lightcone/engine/cloudbuild.py` + (the context stager being replaced). +- Review-verified uv semantics (uv 0.12.3, empirical): `--locked` + errors on stale lock and never re-resolves; `uv run` syncs + additively by default and `--exact` removes extraneous packages; + `--no-sync` silently disables the `--locked` check; a warm read-only + venv runs fine under `--offline`, including git-sourced deps, with + an empty cache; stale-lock detection is a local check; `--no-dev`- + built envs trigger runtime installs under default `uv run` + (motivating the one-environment rule); flipping `[tool.uv] + default-groups` changes the installed set while `uv.lock` is + byte-identical (motivating `install_settings` in the identity); + `uv run` falls back to system PATH for commands absent from the env + and only warns outside a project (motivating the launcher's + discovery + self-verification); uv's project walk-up can resolve a + different project than astra.yaml discovery; uv leaves old env paths + behind when `UV_PROJECT_ENVIRONMENT` moves (motivating gc); venv + recreation on interpreter change deletes marker files; virtual + projects install deps only; `UV_PROJECT_ENVIRONMENT` is honored as + an absolute path, per-project by design; the cache is + concurrency-safe and must share a filesystem with the env for + hardlinking; `--torch-backend` is uv-pip-interface-only; + `required-environments` guards only sdist-less packages; + path/directory lock entries carry no content hash and are + revalidated at run time. +- Ray-vs-dask research (2025–26): Ray uv hook (Ray 2.43+, default-on + later) + open P1s; `working_dir` limits; Ray-on-SLURM community docs + + `ray symmetric-run`; KubeRay maturity vs dask-gateway's ~1 + release/year cadence; Ray Client "experts only"; no Snakemake–Ray + executor exists; resource-semantics parity. diff --git a/docs/design/execution-environment-user-story.md b/docs/design/execution-environment-user-story.md new file mode 100644 index 00000000..0e5c708f --- /dev/null +++ b/docs/design/execution-environment-user-story.md @@ -0,0 +1,412 @@ +# User story: the locked environment is the execution environment + +- **Status:** narrative companion, v3 — non-normative. This document + tells the [execution-environment.md](execution-environment.md) spec + (v6.1: uv-only, sandbox-enforced, full-stack container hatch) as a + sequence of user stories: what a researcher and their coding agent + actually type, see, and get, from first install through laptop → + Perlmutter → hub, in both modes (direct and containerized), + including the failure modes the design converts into pointed + errors. Where this document and the spec disagree, the spec wins. +- **Personas:** + - **Riley** — a cosmology postdoc. Comfortable in a terminal, has + never written a Containerfile, does not want to. Works on a macOS + laptop, runs real jobs on Perlmutter (2–4 nodes), sometimes on + the lab's JupyterHub. + - **The agent** — a coding agent (Claude Code, Codex, …) working in + Riley's project. Per the eval evidence, the agent is the primary + *interface* to lc: it knows uv from training data, it follows + crisp error messages, and its historical failure mode is + environment-boundary confusion. `lc init` scaffolds the agent + notes it needs (the boundary rule, the four-verb map) so it + succeeds without reading the spec. + - **Sam** — a collaborator who receives Riley's repo a year later + and has to trust, verify, and extend the results. + +--- + +## Story 1 — Day one on a laptop + +Riley starts a new weak-lensing analysis on their laptop. + +``` +$ curl -LsSf https://astral.sh/uv/install.sh | sh # if uv isn't there yet +$ uv tool install lightcone-cli +$ mkdir wl-analysis && cd wl-analysis +$ lc init +``` + +That is the entire on-ramp: **uv is the single prerequisite**, and it +is the same tool that installs `lc` (spec G1, §2). No Docker, no +conda, no activation scripts. Bare `lc ` is the one spelling +Riley ever types, on every venue (§4). `lc init` scaffolds +`astra.yaml`, `pyproject.toml` (with `lightcone-cli` as a locked +dependency — the engine is *inside the experiment's lock*), +`.python-version` with an exact interpreter patch, an agent-notes +stanza, and runs `uv lock`. There is no Containerfile in the +scaffold — a direct-mode project never builds an image at all +(G5, §1). + +Riley adds dependencies with the native tool — lc never wraps +dependency management (§1): + +``` +$ uv add numpy scipy astropy +``` + +The agent probes the environment through the one run verb: + +``` +$ lc run python -c "import scipy; print(scipy.__version__)" +$ lc run python src/fit.py --output /tmp/probe +``` + +`lc run` is byte-for-byte the recipe environment — same lock, same +converged `.venv`, and the **same sandbox** recipes get (G4, §4). +The boundary rule the agent is taught (and that `lc init` wrote into +the agent notes) is one sentence: *if a `ModuleNotFoundError` in +this command would mean "fix `pyproject.toml`", it belongs in +`lc run`.* Everything else (git, editors, `lc` itself) stays on the +host. And Riley's old muscle memory is guarded: typing +`lc run best_fit` (the pre-v6 grammar) doesn't exec a mystery shell +command — it errors immediately with *"outputs are materialized, not +run — did you mean: `lc materialize best_fit`?"* (§4). + +When the pipeline is declared in `astra.yaml`, Riley materializes: + +``` +$ lc materialize +``` + +The driver converges the environment once (`uv sync --locked +--exact --compile-bytecode`), starts the run-scoped cluster, and +each rule's recipe runs inside the sandbox with its manifest written +next to its output: + +``` +results/fiducial/best_fit/data.txt +results/fiducial/best_fit/.lightcone-manifest.json +``` + +The manifest records `env_version` (lock + interpreter pin + install +settings + the — here empty — system layer), `git_sha`/`git_dirty`, +the platform, and a `hermeticity` block saying exactly what +enforcement the output ran under — on this macOS laptop: +`{mechanism: seatbelt, fs: declared, network: denied}` (§3, §7). +Inside the sandbox each recipe gets a fresh private `HOME` under the +tmp scope, so matplotlib and astropy work on first import without +ever reading Riley's real dotfiles (§7). + +**What Riley never did:** write a Dockerfile, build an image, +activate a venv, or export an environment variable. + +## Story 2 — The sandbox catches the leak + +Riley's report rule shells out to `latex`. It works — because +`latex` happens to be installed on Riley's laptop via MacTeX. On any +other machine it would fail, and the manifest would have claimed a +pinned environment that silently depended on an undeclared host +tool. This is the #1 leakage channel the design exists to catch +(G6, §7). + +With the sandbox on by default, the recipe fails **at +materialization time, loudly** — and the denial message is the +design's primary UI (§7): it classifies the denial (executable ⇒ +probably a tool; plain file ⇒ probably data), leads with the likely +fix as copy-pasteable TOML or YAML, states the real cost up front, +and keeps the escape hatches in a subdued diagnostics trailer: + +``` +blocked by lc sandbox: cannot execute /Library/TeX/texbin/latex — +not part of the declared environment. + + if this is a tool the recipe needs, declare it in the system layer: + [tool.lightcone.image] + system-packages = ["texlive-latex-base"] + (apt package names — unsure? try: apt-cache search latex) + note: this containerizes the project — podman required (macOS: + one-time `podman machine` VM setup, ~minutes) — and re-stages + all materialized outputs. + + if this is a data file, declare it as an input in astra.yaml. + + diagnostics: lc run --sandbox-debug · lc run --no-sandbox + (recorded as unsandboxed) · lc status +``` + +`--no-sandbox` is never a peer remedy — it's a diagnostic, and using +it is recorded (`hermeticity: {mechanism: none, fs: open}` — never a +silent downgrade). Even when a recipe *swallows* the permission +error and dies with an unrelated traceback, lc appends one fixed +line — *"this recipe ran under the lc sandbox (seatbelt) — try +`lc run --sandbox-debug`"* — so neither Riley nor the agent flails +against an invisible wall (§7). Taking the tool remedy is Story 3. + +The same fence protects the project from the recipe: writes are +scoped to the rule's own `results///` (+ scratch/tmp), so a +misbehaving script cannot clobber sibling outputs, manifests, or +`astra.yaml`. A recipe that legitimately writes intermediates in the +tree declares `sandbox: writable-project: true` on its output in +astra.yaml — a fact in the repo, not a flag lost to shell history — +and its manifest records the honest weaker scope `fs: project-rw` +(§7). + +## Story 3 — The project outgrows PyPI: the container hatch + +Riley pastes the TOML from the error message — and while they're at +it, admits the project also needs R (there's a legacy likelihood +that only speaks it, reached via `rpy2`): + +```toml +[tool.lightcone.image] +system-packages = ["texlive-latex-base", "r-base-core"] +``` + +That declaration *is* the escalation (§1: derived, not configured — +and reversible: delete the list and the project is direct again). +The message already told Riley the two costs: podman (the only extra +install containerized mode ever costs — and the tool a project +headed for HPC or k8s wants eventually anyway), and the blast +radius — `env_version` moves, so **all** materialized outputs go +stale, exactly like any environment edit (§1, §3). + +The next `lc materialize` builds the image and says so: + +``` +building lc-env-9f2c81d44a1b03e7 (first run after an environment +change; ~minutes) +``` + +— `lc run` never builds; it errors with the exact `lc build` +command instead, so a two-second probe never silently absorbs a +build (§4). Inside the image, the apt layer is installed *before* +`uv sync` runs — which is why `rpy2` works: the lock's own packages +build against the declared system layer, not against Riley's bare +laptop (§2). From then on **the image is the execution world**: +driver, workers, recipes, and probes all run from its baked +`/opt/venv`; there is no host `.venv` at all (§1). The container +bounds the world to the project + declared inputs, and inside it the +same Landlock shim scopes each recipe to its own output dir — on +macOS too, since the podman VM is Linux — so the manifest says +`{mechanism: podman+landlock, fs: declared, network: denied}` (§7). + +Anyone — Riley included — can check what state the project is in: + +``` +$ lc status +mode: containerized (2 system packages) +image: lc-env-9f2c81d44a1b03e7 — built (digest sha256:4c1f…) +sandbox: podman+landlock (fs: declared, network: denied) +``` + +Everything else in Riley's muscle memory is unchanged: `uv add` +still manages Python deps; `lc run`, `lc materialize`, `lc status`, +`lc verify` behave identically. Rebuilds happen only when the +environment changes — never on code edits (G5). The honest residue +is stated, not hidden: apt is name-pinned, so the build records a +dpkg snapshot (content-hashed into the manifest), and the macOS VM +means linux builds and no GPU — `platform` attested from inside the +boundary, which is where everything now runs (§3, §5). + +## Story 4 — Same project family, Perlmutter + +The weak-lensing project from Story 1 — still direct mode, pure +PyPI — needs 2–4 GPU nodes. The venue change is a clone and an +allocation (§5): + +``` +perlmutter$ uv tool install lightcone-cli # once +perlmutter$ cd $CFS/myproj && git clone … wl-analysis && cd wl-analysis +perlmutter$ salloc -N 2 … +perlmutter$ lc materialize +``` + +The project lives on **CFS, not `$HOME`** — CFS is writable from +compute nodes, so environment convergence works mid-allocation. The +cache placement (`$SCRATCH/uv-cache`, copy link-mode) comes from +lc's site registry and is injected by the launcher after the ambient +scrub — Riley exports nothing (§4). The env is the same in-tree +`.venv` as the laptop. + +The execution discipline is the spec's one sentence: **converge +once, then never write to the environment** (§6). Every job runs +`uv run --no-sync` with the offline overlay, so a hundred workers +never race installs onto Lustre — a warm env is a no-op check, drift +is a loud failure. Each recipe is wrapped in **podman-hpc** using +the venue's static runtime image, with the honest mount set — +project **read-only**, own output dir RW, declared inputs RO — so +the manifest's `fs: declared` means the same thing here as +everywhere else (§7). The one venue chore is fetching that static +image once per login node era; forget it and the compute-node +preflight prints the exact login-node command instead of a podman +error (§5). + +**Mid-run edit, caught.** While rules are running, the agent +helpfully runs `uv add emcee` in another pane. The next rule's +pre-gate re-hashes the lock against the run-start `env_version` and +fails with *"lock changed mid-run — re-run lc materialize"* (§6). + +(The containerized R project of Story 3 can come here too — its +image built and migrated once on a login node, its workers launched +inside it — but that path rides the podman-hpc full-stack spike, and +until the GPU question clears, lc says so plainly instead of +pretending: §5.) + +## Story 5 — Same project, the hub + +Riley opens the lab JupyterHub, clones the direct-mode repo into NFS +home, and runs `lc materialize` from the notebook pod. What does +*not* happen: no Cloud Build job, no per-project image, no waiting +for a build-and-push before the first run. + +Worker pods run a **static, deployment-managed runtime image** +(referenced by digest; slim base + uv + the exec-shim). Riley's +environment, interpreter, and code all ride the NFS home the pods +already mount; lc passes the env location as `LC_PROJECT_ENV` +through the standard cluster options, and the shim execs from it — +failing loudly if it's unset or absent, never falling back to the +image's Python (§5). + +Honesty in the manifest: the pod bounds only the OS layer, so +hermeticity is recorded as `fs: os-only` — upgraded to `declared` +only where in-pod Landlock is actually enabled, and network recorded +`allowed` (lc cannot observe NetworkPolicy) rather than pretended +(§7). This is also where G4's one documented exception lives: +notebook-pod probes share the lock but not the workers' OS layer. + +(The containerized project runs here with its **own image as the +worker pod** — full stack, built by Cloud Build from three files, +never containing code — and the deployment derives the notebook +image `FROM` the project image, which closes even the G4 exception +on this venue: §5.) + +**Editing code between runs costs nothing.** Riley edits +`src/fit.py`, re-runs `lc materialize`, and only the affected rules +re-execute — a code edit never triggers an image build in either +mode, because no image ever contains project code (G5). + +## Story 6 — The environment changes; the blast radius is surfaced + +Riley upgrades numpy: + +``` +$ uv lock --upgrade-package numpy +$ lc materialize +environment changed: 14 materialized outputs are now stale +``` + +`env_version` moved, so `code_version` moved, so Snakemake's rerun +triggers fire — the one case where re-materialization is *correct*, +announced at decision time rather than discovered later. In direct +mode there is no image rebuild and no tag bookkeeping — the identity +is the lock itself; in containerized mode this is the one moment a +rebuild happens, which is exactly when it's meaningful (G3, G5, §3). +And a rebuilt image is not trusted by name: the run pins the digest +the driver resolved, and every worker asserts it — two nodes can +never silently run different userlands under one tag (§3, §6). + +The same identity machinery refuses the unauditable: a path +dependency (`uv add ../my-hack`) is rejected at lock-scan time — +except the project's own package, which is exempt (§3). Non-default +dependency groups draw an advisory ("outside lc's guarantees"), and +the sandbox — whose allowlist contains only the project environment — +is what makes that rule real rather than documentation (§7). + +## Story 7 — A year later: Sam verifies, reproduces, extends + +Sam clones the repo (results synced alongside) and, before trusting +anything: + +``` +$ lc status # offline, local-only — mode, image state, sandbox +$ lc verify +``` + +`lc verify` recomputes each output's `data_version` and walks the +provenance chain; failures surface as `tampered_data`, +`broken_chain`, or `missing_manifest` — and it surfaces +**dirty-tree** and **unsandboxed** outputs distinctly (§3). Sam can +see, per output, not just *what* environment produced it but *what +enforcement it ran under*: an output with `hermeticity: {mechanism: +landlock, fs: declared}` earns different trust than one with +`{mechanism: none, fs: open}` — or the honest middle ground +`fs: project-rw`. For a paper's final runs, Sam's CI adds +`--require-sandbox=declared-fs`, refusing any output below that bar +(§7). Outputs written under an older manifest schema report +`pre_migration` under the `SCHEMA_VERSION` bump, distinct from +tampering. + +Reproducing is the same on-ramp as Story 1: install uv (plus podman +if the project declares a system layer — the mode is derived from +the files, Sam doesn't have to guess), `lc materialize`. The +lockfile *is* the environment definition; the system layer is one +TOML list; even the dpkg snapshot of the image that built the +figures is content-hashed in the manifests, so the audit outlives +any image registry. The claim Sam gets is the honest one the spec +commits to: **pinned environment identity, never bit-identical +outputs** (G3) — BLAS dispatch and thread scheduling still vary by +hardware, and the manifest's attestation fields say so instead of +pretending. + +## Story 8 — Migrating the existing project + +Riley's older project has the v3-era scaffold: an authored +Containerfile and image tags in its manifests. `lc init` on it +converges the scaffold — and on finding the authored Containerfile +it **refuses with instructions** rather than hiding consent behind a +flag: *"found an authored Containerfile; v6 generates images from +the lock — delete or rename it, then re-run `lc init`"* (§8). Old +outputs are not invalidated wholesale: `lc verify` reports them as +`pre_migration`, distinct from tampering. New runs write current +manifests; the two coexist in one results tree. + +Containers in Riley's life are now generated, never authored: the +hatch's derived image (§3) for declared system layers, and — for +truly arbitrary cases — the digest-pinned **BYO** per-output +container (§8), declared in `astra.yaml`, hashed into identity, the +one container lc does not generate. + +--- + +## What the stories never contain + +The negative space is the design (§8 deletion ledger): + +- No second package manager: uv is the only substrate, on every rung + of the ladder. +- No authored Containerfile, ever — images exist only for projects + that declare a system layer, are generated from the lock, tagged + by content, digest-pinned at run time, and never contain project + code. +- No container runtime on any direct-mode laptop — Landlock and + Seatbelt deliver the recipe fence with zero install; podman + appears only when a project's dependencies leave PyPI. +- No dual environments: a containerized project has no host `.venv` + — the image is the execution world. +- No environment verbs, placement tiers, markers, or leases — the + env is in the project tree (or the image) on every venue. +- No `requirements.txt`, no activation, no hand-exported venue + variables. +- No flags for one-time events or repo-external behavior changes — + migration consent is a file operation; writable-project is an + astra.yaml declaration. +- No one-way doors: the escalation is a declaration, and deleting it + de-escalates. +- No pretending: every place enforcement or identity is weaker (hub + pods without Landlock, Landlock's missing network control, apt's + name-only pinning, `project-rw` scopes, sdist builds, host GPU + drivers), the manifest records the truth — and a downgrade prints + a console line, never just a field (§7). + +## Traceability + +| Story | Spec anchor | +|---|---| +| 1 — day one | G1, §2 (uv project, agent notes), §4 (launcher, `lc run`, rename guard), §3 (manifest) | +| 2 — sandbox denial | G6, §7 (policy, HOME/XDG, denial UX, failure trailer, writable-project key) | +| 3 — container hatch | §1 (full-stack ladder, blast radius), §2 (system layer), §3 (image identity, dpkg residue), §4 (build moment, `lc status`), §7 (in-container Landlock) | +| 4 — Perlmutter | §4 (site registry), §5 (venue row, static-image preflight), §6 (discipline, mid-run gate), §7 (honest mount set) | +| 5 — hub | G4 exception, §5 (static image + `LC_PROJECT_ENV` / project-image pods + notebook-FROM contract), §7 (`os-only`, network honesty) | +| 6 — env change | G3, G5, §3 (identity, digest pinning, lock scan) | +| 7 — verify | §3 (attestation, schema), §7 (`hermeticity`, `--require-sandbox`, downgrade notice) | +| 8 — migration | §8 (Containerfile refusal, BYO container), §11 (stages, `pre_migration`) | diff --git a/docs/design/execution-environment-v6-review.md b/docs/design/execution-environment-v6-review.md new file mode 100644 index 00000000..0becb72b --- /dev/null +++ b/docs/design/execution-environment-v6-review.md @@ -0,0 +1,95 @@ + + +# v6 spec review — synthesis + +## Executive summary + +The v6 architecture — one uv substrate, mode derived from declarations, OS-sandbox-by-default with a generated-container hatch, honest per-output enforcement records — is judged sound and well-evidenced by all four lenses and both judges; nobody asks for architectural change. Both judges score it **6/10** and both withhold sign-off pending edits, because the spec currently fails its own promises in three places. First, containerized mode as written deadlocks on its own §2 advertisement: the host-side `uv sync --locked --exact` must build the entire lock on the bare host, so lock-level system deps (libhdf5-dev, r-base-core for rpy2) fail before the image is ever used. Second, the flagship venue is internally dishonest: Perlmutter direct mode mounts `$PWD` RW yet stamps `fs: declared`, and the user story calls project-RW "the normative mount set". Third, the companion narrative silently un-containerizes Riley's project between Stories 3 and 4, and the first thing every persona types (`lc` vs `uv run lc`, `lc run` vs `lc materialize`) is ambiguous or booby-trapped. The software judge additionally surfaced two load-bearing gaps no lens caught: workers are never required to run the digest the driver resolved, and the Landlock-FD-through-`uv run` inheritance that the entire Linux direct mode rests on is unverified. Every fix lands inside the existing structure; approve after the must-fix list below. + +## Must fix + +- **Close the containerized host-sync deadlock** — §1/§2/§4 (launcher step 4)/§6; robustness F1 (confirmed), both judges' #1. The host-side `uv sync --locked --exact` (driver preflight, worker env check) must build the entire indivisible lock on the bare host, so the hatch's own example (`libhdf5-dev`, `r-base-core`) — exactly what lock-level sdist builds need — dead-ends before the image runs. Make the restriction normative ("the system layer satisfies only tools recipes invoke; every locked package must sync on the bare host"), enforce it at escalation time with a pointed error naming the BYO per-output container as the escape, and fix the §2 example list — or explicitly redesign S2/S3 to run in-image on every venue as a §12-acknowledged change. + +- **End the Perlmutter-direct `fs: declared` dishonesty** — §7 policy prose vs matrix "Perlmutter, direct" row; UX-5, robustness F2, consistency F3 (all confirmed), both judges. The row mounts `$PWD` RW — sibling outputs, manifests, astra.yaml, .venv all writable — yet stamps the same `declared` label a genuinely scoped Landlock run earns, breaching both "one policy, both modes" and the never-pretend rule. Give the direct wrap the normative mount set (project RO, own output dir RW, interpreter dir RO — podman-hpc already expresses this in the containerized row), or record an honest weaker value (e.g. `project-rw`); never stamp `declared` on whole-tree-RW. + +- **Make the Perlmutter direct wrap coherent: provisioning, §1 exception, spike contingency** — §1, §5, §7 matrix, §11 step 3; UX-10 (confirmed), overengineering F1 (revised), both judges. Nobody is told who migrates the static runtime image (Story 4 even promises "no images to pre-build"), §1's "no container anywhere" omits this second static-image use, and the wrap coexists unannotated with its Landlock replacement. Add a direct-mode materialize preflight with the exact login-node command (mirroring the containerized column's pattern), reword §1's exception to cover both static-image uses, and mark the wrap-vs-Landlock pieces (step-3 image role, step-5 direct mount set) explicitly spike-contingent with the deletion path pre-declared. + +- **Repair the user-story arc and purge v5 residue** — user-story Stories 3–5, 7, 8; consistency F1, F2, F5, F10 (all confirmed), both judges. Story 3 containerizes Riley's project, then Stories 4–5 narrate it as direct mode ("no images to pre-build", "this direct-mode project"), Story 4 calls project-RW "the normative mount set", and "UV_*/PIXI_*" scrub, "new in v5", and "v5 manifests" are all deleted-machinery residue. Either continue the containerized project (login-node build/migrate in Story 4, project-image pod in Story 5) or explicitly rewind to a direct-mode project; fix the mount-set parenthetical to match §7; delete `/PIXI_*`; replace v5-as-manifest-format with SCHEMA_VERSION language. + +- **Refuse packaged projects in containerized mode** — §2 vs §11 step 6, §6 step 2; robustness F5 (confirmed), both judges. The image is built `--no-install-project`, so a packaged project's escalation produces a container in which `import my_analysis` fails — the design's founding failure mode returned via its own hatch, with v4's `lc doctor` guard deleted and unreplaced. Refuse the combination at mode-detection time with BYO named as the escape, and state the hub env-check semantics against the code-free image env. + +- **Specify HOME/XDG inside both boundaries** — §7 filesystem policy and mount set; robustness F3 (confirmed), both judges. `$HOME` is neither readable nor writable and HOME/XDG_* are undefined, so matplotlib/astropy/R break on first import — and the obvious implementer patch ($HOME RO) reopens the dotfile-steering channel G6 exists to close. Normatively set HOME, XDG_CONFIG_HOME/XDG_CACHE_HOME/XDG_DATA_HOME, and MPLCONFIGDIR to a fresh per-recipe directory under the writable tmp scope, both modes. + +- **Add the ELF loader to the exec allowlist** — §7 two-tier exec; robustness F7 (confirmed), both judges. Landlock checks EXECUTE on the ELF interpreter's open, so as written `bash` and `.venv/bin/python` both fail EACCES on `/lib64/ld-linux-*` — the specified sandbox is unusable on every Linux venue. Add the resolved loader path(s) (glibc/musl, realpath'd) to the tier, note shared libraries need only the read baseline, and add a policy unit test exec'ing a dynamically linked binary. + +- **Guarantee a sandbox trailer on every nonzero sandboxed exit** — §7 mandatory rules; UX-6 (confirmed), both judges. The crisp denial fires only when errno + path extraction succeed; a recipe that swallows PermissionError leaves the user (and agent) with a bare traceback and no hint a sandbox was active — the chmod-flailing failure mode. Specify a fixed trailer on every failed sandboxed run naming the mechanism and pointing at `lc run --sandbox-debug`. + +- **Guard the `lc run` rename** — §4, §11 Stage 1 vs today's shipped `lc run [outputs...]`; UX-7 (confirmed), both judges. Post-migration, trained users and agents typing `lc run best_fit` get the output name exec'd as a shell command — "command not found" for a correct old habit; §11's "hint" is unspecified. Specify the collision guard: a first arg matching a declared output errors with the `lc materialize` redirect before any exec, and bare `lc run` prints one line announcing the recipe shell. + +- **One command spelling everywhere** — §4, user-story Stories 1 and 4; UX-1 (confirmed), both judges. §4 crowns `uv run lc` canonical while every laptop story types bare `lc`, and Story 4 switches spellings silently — plus `uv run lc init` cannot work pre-lock. Make bare `lc ` the single documented form, demote `uv run lc` to an equivalence parenthetical, and fix Story 4 (add the one-time `uv tool install` line on Perlmutter, or one sentence explaining uv-run there). + +- **Rebuild the denial message — the design's primary UI** — §2, §7; UX-2 (revised), UX-3 (confirmed), both judges (with the software judge's hint-table cap). Today's message is a three-way exam with `--no-sandbox` as the cheapest answer, an unassisted path→apt-name puzzle, a surprise podman-machine VM discovered one failed run later, and an unspecified raw-apt-error surface for wrong package names. Order remedies by best-guess classification (exec-bit/bin-dir heuristic), demote `--no-sandbox`/`--sandbox-debug` to a subdued diagnostic trailer, give the ASTRA-input remedy a copy-pasteable snippet matching the TOML one, state the machine's real cost (podman VM setup on macOS) in the first denial, parse apt "Unable to locate package" into a pointed answer, and ship a *capped, versioned* tool→package hint table (or a generic `apt-cache search` line) rather than an open-ended mapping. + +- **Forward-looking visibility in `lc status`** — §1, §4; UX-4 (confirmed), both judges. No surface reports mode, image tag/built-state, or this-host sandbox mechanism; every "will this build?/is this sandboxed here?" question requires reading pyproject plus the spec. Add three header lines to `lc status` (mode, image + built-state, this-host mechanism/postures), point the denial and podman-missing errors at it, and restate the status invariant as "offline, local-only" since it now reads pyproject and the image store. + +- **Split the build moment: materialize builds, run errors** — §4 step 4, §6; UX-12a (confirmed), both judges. "Builds or errors-with-the-exact-command" is ambiguous, so a two-second probe can silently absorb a multi-minute image build. `lc materialize` builds (matching §6's existing preflight text); `lc run` errors with the exact `lc build` command; any building verb prints "building lc-env- (first time after an environment change; ~minutes)". Fix Story 3's "on the next lc run" accordingly. + +- **Specify the macOS podman-machine mount-source preflight** — §5, §7, §11 step 6 + spike list; robustness F6 (revised), both judges. A declared input outside the VM's shared directories (`/Volumes`, `/opt`) becomes a silently empty mount — a zero-file glob "succeeds" and stamps `fs: declared`. Fill in §11 step 6's existing preflight item: verify every mount source lies under the machine's shared mounts, error with `podman machine set --volume`/relocation as remedy; add path coverage to the spike checklist and soften §7's "on every venue including macOS". + +- **Resolve UV_CACHE_DIR ownership on Perlmutter** — §4, §5 vs Story 4; UX-9/robustness F9/consistency F12 (all revised), both judges. "Re-injected venue-managed values" never names the source (site registry vs preserved user export), and Story 4's manual export is unexplained ritual-or-load-bearing. State that site_registry supplies `$SCRATCH/uv-cache` + `copy` (re-injected post-scrub for all lc-initiated uv calls, ambient value honored as override), and either delete Story 4's export or add the one clause explaining it covers the pre-scrub outer invocation; also promote the writable-project escalation out of its parenthetical and name its astra.yaml field. + +- **State the escalation blast radius at the ramp** — §1, Story 3; product judge addition (no lens finding), demanded as blocking. Declaring `system-packages` moves `env_version` and stales every materialized output, not just the rule that needed TeX — Story 6 discloses this cost for a numpy bump but the hatch itself is silent. Add the one sentence in §1 and Story 3 where the escalation is introduced. + +- **Pin image execution to the driver-resolved digest** — §6 job command, run_rule; software judge addition (no lens finding), demanded as blocking. Name-pinned apt means two builds of one tag legitimately differ, so tag-addressed execution can run different userlands across nodes within a single run while every manifest looks consistent — the image-side twin of the mid-run relock gate. Pass the resolved digest in the job command and assert it in run_rule. + +- **Verify Landlock-FD inheritance through `uv run`** — §7, §13; software judge addition (no lens finding), demanded as blocking. The entire Linux direct-mode sandbox assumes the ruleset FD survives uv's spawn/exec of the shim; this is in no verified-semantics list and a Landlock FD cannot be reopened. Add the empirical entry to §13 or the spike, and state the fallback (exec the resolved interpreter directly for step 3) if uv drops non-stdio FDs. + +- **Editorial and manifest-precision sweep (one commit)** — both judges demand the batch. Qualify G4 with the hub notebook-probe exception and align Story 3 + tradeoffs wording (consistency F4); drop or define `export` (F6 + UX-12b); spell out "step 3 (the recipe exec)" for both "S3" uses and link hermeticity-enforcement.md by name (F7); back or soften the "verified on uv 0.12.3" flags-beat-ambient claim (F8); expand WRROC on first use (F9); make `dpkg_snapshot` a content hash so the attestation outlives image GC (overengineering F4 revised); state which attestation fields are host- vs boundary-scoped when `worker_runtime: container`, with the prelude or derived-from-digest choice named, and fix §5's "platform attested honestly" (robustness F4 revised); add the normative network-enum mapping paragraph — denied = non-loopback blocked, hub workers record `allowed`, value derives from flags actually applied (F11 revised); annotate the Perlmutter network cells as provisional on the `--gpu`/`--net` spike with a stated fallback posture (robustness F8 revised); add `--compile-bytecode` to the direct-mode converge and shrink the `__pycache__` caveat to project-tree sources (UX-11 revised); specify `lc build` on a direct-mode project as an explanatory no-op (UX-12c); document the probe's write scope as tmp/scratch only (judge-added). + +## Should fix + +- **Console notice on enforcement downgrade** — §7; product judge (missed). When the probe finds `mechanism: none`, the manifest records it but nothing prints — a user completes a materialize believing they were sandboxed. Extend probe-record-never-silent to include one console line per downgraded run. + +- **Shim-to-engine delegation compatibility contract** — §4; software judge (missed). The tool-env launcher (any version) execs a project-locked engine that may be years old, with `LC_DELEGATED=1` as the only specified interface; every launcher change is a potential silent break for old locked projects. Declare the delegation boundary minimal-and-frozen or add a version handshake. + +- **Containerfile ENV ordering** — §11 step 6; software judge (missed). The baked `UV_OFFLINE=1` would break the image's own network-needing `uv sync` layer if emitted first. Specify build-args-then-final-ENV ordering in the generator spec and pin it in the golden tests. + +- **Test fixtures for the denial-UX fallback** — §11 test list; software judge (missed). Nothing tests the swallowed-PermissionError trailer or the re-stat classifier against rewrapped errors — the two cases where agents actually flail. Add both as fixture recipes in the Landlock/Seatbelt suites. + +- **Venue-mechanism regression canary** — §7; software judge. The podman-hpc and GKE-seccomp behaviors are one-time spikes but NERSC/GKE upgrade underneath the matrix. Add a maintenance note (periodic canary materialize per venue, or revalidate-on-site-change), mirroring the exec allowlist's "maintained policy surface" framing. + +- **Honest on-ramp: install uv in Story 1** — user story; product judge (missed). The flagship story presumes uv is installed; the actual first command appears only in Story 7. Add the one curl/brew line. + +- **Surface-without-story audit** — user story; product judge (missed). Containerfile.extra, BYO containers, the writable-project escalation, per-output `network:`, bare `--require-sandbox`, and `lc build` are exercised by no story. Have the companion exercise (or drop mention of) whatever surface survives the pruning below. + +## Simplifications + +- **Drop per-output `network:` from the normative surface** — §7, §11 step 8 → §12 only; overengineering F5 (confirmed), both judges. The spec commits ASTRA schema churn for a mechanism §12.3 admits is unfinished. Remove the opt-in and the migration step; v1 remedies are declare-the-download-as-input or recorded `--no-sandbox`; note in §12.3 that `--no-sandbox` being all-or-nothing motivates finishing the design. + +- **Replace `--accept-containerfile-loss` with a refusal message** — §11 Stage 2, Story 8; overengineering F6 (confirmed), both judges. A permanent CLI flag for a one-time migration event. `lc init` refuses with "delete or rename the Containerfile, then re-run" — the user's own file operation is stronger consent and zero forever-surface. + +- **Delete the `--sandbox-writable-project` flag; keep only the ASTRA-declared spelling** — §7; UX-8 (confirmed), both judges. A behavior-changing flag lives outside the repo, so a year-later clone cannot reproduce the run from files. Keep declarations (versioned, reviewed); name the astra.yaml field and specify what `hermeticity.fs` records for such runs. + +- **Containerfile.extra: keep the null-hashed schema slot; no silent half-state** — §1–§3, §11 step 6; overengineering F2 (revised) as amended by the software judge. The slot stays in the env_version formula (removal would move every project's identity), but a declaration the mode-derivation honors and the generator ignores is banned half-state. Either implement the rendering fully (it is one stage plus an existing hash input) or make its presence a crisp refusal until the §12.1 survey enables it. + +- **Compress the triplicated pixi arguments — keep the §13 evidence block** — status header, §10, §13; overengineering F7 (revised), non-blocking per the product judge. The deciding arguments appear three times; the empirical pass appears once, and that once is a recorded decision. Cut the status header to the decision-record link and shrink §10's bullet to "rejected — see substrate-default-tradeoffs.md"; never delete the §13 block as-is (relocate only in a change that updates both docs' cross-references). + +## Considered and rejected + +- **Consistency-lens F3 (hub exec-shim duplicates uv-run)** — refuted: the verified uv evidence says the opposite of the premise (`uv run` silently falls back to system PATH; `--no-sync` disables the locked check); the shim's fail-loud contract is load-bearing and stays. +- **Overengineering F4 original (delete `worker_runtime`, `exec_allowlist_version`)** — overruled: `worker_runtime` is not derivable from mechanism (Perlmutter-direct wraps only the recipe exec while the worker is host-side), and the version-stamped allowlist keeps audits self-contained; only the dpkg_snapshot clarification survives. +- **Overengineering F7 original (delete the §13 pixi empirical block)** — overruled: it is the sole authoritative copy and its retention is itself a recorded decision in the tradeoffs doc. +- **Overengineering F1 original (delete the podman-hpc direct wrap, gate everything on the Landlock spike)** — overruled: SLE-15 Landlock inclusion is unverified and Lustre has no field reports, so the evidence-backed wrap stays the default; only the honesty/annotation fixes survive (in Must fix). +- **Overengineering F2 original (delete Containerfile.extra from schema and formula)** — overruled: removing then re-adding the hash input would move env_version for every project; the scoped "root Containerfile never user-authored" rule is not violated, and BYO forfeits G4 probe fidelity. +- **UX-2 original ("lc knows the denial type")** — overruled to heuristic ordering: Landlock returns one errno for read and exec denials, and the mandatory-rules doc requires the escape hatches to remain visible in the message. +- **UX-3 partial (open-ended path→apt hint table)** — capped by the software judge: an unbounded mapping is a forever-dataset that drifts with every Debian release; ship a small versioned table or a generic `apt-cache search` line. +- **UX-9/robustness-F9 original contradiction framing** — weakened: the user export is load-bearing for the pre-scrub outer `uv run`, so the two mechanisms coexist; only the ownership-clarity fix survives (in Must fix). +- **UX-11 original (delete the `__pycache__` caveat)** — weakened: project-tree source `__pycache__` writes remain denied under project-RO, so the caveat shrinks but cannot vanish. +- **Robustness F8 original (false network attestation on GPU rules)** — weakened: three explicit mandates already require recording what actually ran; only the provisional-annotation hygiene survives (in the sweep). +- **UX-1 original (delete `uv run lc` outright)** — refined: it is the zero-shim path on venues without `uv tool install`; demote to a parenthetical, don't delete. +- **F11 original ("enum cannot express the postures")** — weakened: deny-with-loopback is the spec's own normal meaning of `denied`; only the hub allowed-vs-denied mapping was genuinely unspecified (fixed in the sweep). diff --git a/docs/design/execution-environment.md b/docs/design/execution-environment.md new file mode 100644 index 00000000..72967651 --- /dev/null +++ b/docs/design/execution-environment.md @@ -0,0 +1,818 @@ +# Spec: the locked environment is the execution environment + +- **Status:** specification, v6.1 (uv-only, sandbox-enforced, + container hatch — full-stack). Normative. v6 replaced v5's + dual-substrate layer with a single-substrate ladder: uv alone by + default, hermeticity via OS sandboxing, and an on-demand container + hatch (decision record: + [substrate-default-tradeoffs.md](substrate-default-tradeoffs.md)). + **v6.1 resolves the 2026-08 multi-agent review** + ([execution-environment-v6-review.md](execution-environment-v6-review.md)): + containerized mode now runs the **entire execution stack inside the + image** — engine, workers, recipes, probes — closing the review's + host-sync deadlock (a bare host can never be asked to build + lock-level system dependencies); the Perlmutter direct wrap is + brought to the honest mount policy; image execution is + digest-pinned; the sandbox policy gains the ELF loader and a + per-recipe HOME/XDG contract; the denial message, `lc status` + visibility, command spelling, and `lc run` rename guard are + specified as the primary UI they are; and the review's + simplifications land (per-output `network:` deferred, two flags + deleted). Scale (>~10 nodes) remains out of scope. + **Implementation record (2026-08):** the first implementation round + deviates from this spec in three recorded ways — (1) ASTRA carries + no container/sandbox keys at all (the `sandbox: writable-project` + escalation lives in pyproject under `[tool.lightcone.sandbox]`, and + BYO per-output containers are deferred); (2) venue scope is + local-only with podman as the sole container backend (Perlmutter, + hub/GKE, and Cloud Build return behind the Builder/runtime/boundary + protocols); (3) §7 gains the `PYTHONPYCACHEPREFIX` amendment noted + in place. Evidence: + [execution-environment-rationale.md](execution-environment-rationale.md), + [uv-vs-pixi-adoption.md](uv-vs-pixi-adoption.md), + [hermeticity-enforcement.md](hermeticity-enforcement.md), + [environment-substrate-evaluation.md](environment-substrate-evaluation.md). +- **Scope:** CLI surface, `lc init` scaffold, engine environment + handling, the container hatch (`lc build`, generated Containerfile, + image identity), the dask fabric's environment contract, recipe + sandboxing, eval prompt. Not in scope: executor scheduling + semantics; hub charts (they consume the §5/§7 contracts); native + Windows (WSL2 is supported via the Linux path); >~10-node scale. +- **Goals:** (G1) start with uv alone; a project adds exactly one + more tool (podman), and only when its dependencies leave what uv + can lock; (G2) uniform execution laptop → Perlmutter (2–4 nodes) → + hub/GKE; (G3) honest pinned environment identity — *pinned + identity, never bit-identical outputs*; (G4) probing is + byte-for-byte the recipe environment, sandbox and container + included — one documented exception: direct-mode hub notebook + probes share the lock but not the OS layer (§5); (G5) code edits + never trigger an image build; environment edits rebuild only + containerized projects' images — exactly when a rebuild is + meaningful; (G6) recipes mechanically cannot use tools or files + outside the declared environment wherever an enforcement mechanism + exists — and every output records exactly what enforcement it ran + under. + +## The design in one paragraph + +A project is `pyproject.toml` + `uv.lock` + `.python-version` — uv is +the only substrate. In **direct mode** (the default) the environment +lives in the project tree on every venue (`.venv`); no image is ever +built; every recipe runs inside a **sandbox restricted to the +declared set** — Landlock (Linux), Seatbelt (macOS), a podman-hpc +wrap (Perlmutter), the pod's OS layer (hub). When a project needs a +dependency uv cannot source — R, Julia, TeX, MPI, compilers, system +libraries a locked package links against — it declares a **system +layer** (the `[tool.lightcone.image]` table, optionally +`Containerfile.extra`), which flips the project into **containerized +mode**: lc renders the locked environment *plus* the system layer +into a content-addressed image, and from then on **the image is the +execution world** — driver, workers, recipes, and probes all run from +the image's baked environment; no host environment exists for the +project at all. Because `uv sync` runs inside the image build, after +the apt layer, lock-level system dependencies (rpy2 needing R, sdist +builds needing headers) resolve where the system layer actually is. +The container is a *cache of the lock plus the declared system +layer*, never a definition; project code never enters an image; code +reaches every venue through the filesystem the venue already shares. +One execution discipline in both modes: converge once (sync the +`.venv` / build the image), then execute without writing to the +environment. The manifest chain is unchanged in shape: `env_version` +(covering the system layer) sits inside `code_version`; a double +mid-run relock gate ties every recipe to the lock live at run start; +image execution is pinned to the digest the driver resolved; a +`hermeticity` field records mechanism, file scope, and network +posture per output. + +## 1. The environment ladder + +Two modes, one substrate. Mode is **derived, not configured**: +declaring a system layer *is* the escalation. + +- **direct** (default): no per-project image, and no container on + laptops. (Two venue-provided static images exist in direct mode and + are not project artifacts: the hub's runtime image, and the same + image used by Perlmutter's recipe wrap — §5, §7.) +- **containerized**: triggered solely by the presence of + the `[tool.lightcone.image]` table (or `Containerfile.extra`). + Per-project, never per-output. **Full-stack**: every process that + touches the run — driver engine, dask workers, child snakemake, + recipes, `lc run` probes — executes inside the project image from + its baked `/opt/venv`. There is no host `.venv`; the host keeps + only the thin launcher (§4). One environment, one place; engine + version coherence is preserved because everything runs from the + image = the lock. + +**Rules that keep it thin (normative):** + +- **lc never wraps dependency management.** `uv add` is uv's + business; declaring the image is one TOML table (§2); lc wraps + execution and identity only. +- **The nudge is the sandbox denial.** Direct mode's denial UX (§7) + is where users meet the hatch. lc never escalates silently. +- **Escalation is reversible, and its cost is announced.** Deleting + the declaration returns the project to direct mode. Declaring it + moves `env_version` — **every materialized output goes stale, not + just the rule that needed the tool** — and lc says so at + escalation time, exactly as it does for any environment edit (§3). + De-escalation leaves the old image inert; reclaim with + `podman image prune` (an lc-side image GC verb is a §12 re-add + candidate). +- **The root Containerfile is never user-authored.** The image is + generated (§3); `Containerfile.extra` is the bounded escape — a + stage `FROM` the derived image, content-hashed into identity, and + **fully rendered by the generator** (a declaration the mode + derivation honors but the generator ignores is banned half-state). + The **BYO digest-pinned per-output container** remains the + schema's escape for truly arbitrary cases (§8). +- **Packaged projects are refused in containerized mode.** The image + is built `--no-install-project` (code never enters an image), so a + packaged project's `import my_analysis` would fail inside its own + container — the design's founding failure mode. `lc` refuses at + mode-detection time: "containerized mode requires a virtual + project (no `[build-system]`) — restructure, or use a declared + per-output container (BYO)." + +## 2. Environment definition + +**The uv project (both modes).** `pyproject.toml` + +`.python-version` (exact interpreter patch) + `uv lock`; +`[tool.uv] required-version` and `required-environments` scaffolded; +`lightcone-cli` in dependencies (lock pins exact — the engine is +inside the experiment's lock). One environment, no group/feature +splits (identity hashes the install-selection settings, §3; extra +groups draw an advisory: "outside lc's guarantees"); virtual by +default — packaged projects are supported in direct mode via +editable install (the project's **own package is exempt** from the +path-dep refusal; everything else path-like is refused) and refused +in containerized mode (§1); GPU via pinned indexes +(`[tool.uv.index]` + `[tool.uv.sources]`), host driver attested; +bootstrap = uv alone. `lc init` also scaffolds an **agent notes +stanza** (AGENTS.md, or appended to an existing one) carrying the +boundary rule — *"a `ModuleNotFoundError` under `lc run` means fix +`pyproject.toml` with `uv add`, never install into another env"*, +the containerized-mode habit (*"`uv add` runs on the host, bare — +add `--no-sync` in containerized projects; never `lc run uv add`"*), +and the four-verb map (`lc run` probes, `lc materialize` executes, +`lc status` reports, `lc verify` audits), so the design's primary +interface works without reading this spec. + +**The image declaration (containerized mode).** One TOML table is +the whole user-facing surface of the container hatch; its presence +is the escalation (§1), and every key is hashed into `env_version` +and the image tag: + +```toml +[tool.lightcone.image] +# base — optional. Digest-pinned OCI ref replacing the generator's +# default base (an engine constant). The canonical use is a vendor +# userland, e.g. NVIDIA CUDA: +base = "nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:9f2c81d4…" + +# system-packages — apt package names, installed before uv sync. +system-packages = ["texlive-latex-base", "r-base-core", "libhdf5-dev"] +``` + +- **`system-packages`** (list of apt names, sorted into the + identity). Both classes of system dependency are supported, + because the environment is built *inside* the image after the apt + layer: tools recipes invoke (`latex`, `Rscript`, `julia`) **and** + libraries locked packages need at build or import time + (`libhdf5-dev` for an sdist h5py, R for rpy2). The docs still + steer users to check PyPI first — h5py wheels bundle libhdf5, + NVIDIA ships `nvcc`, BLAS rides inside numpy — the hatch is for + what genuinely has no wheel. +- **`base`** (optional digest-pinned OCI ref). A tag-only ref is + **refused** — "pin the digest" — because the image must remain a + pure function of the repo plus the engine (the unpinned-base + identity hole is what killed the v1 design). Registry + authentication is the venue's (`podman login`, Cloud Build's + service account); lc never stores credentials. GPU guidance: + the default base + PyPI CUDA wheels is the first choice; a + vendor `base` is the step up when a vendor userland is genuinely + required. + +**The base contract** (the Modal lesson: base flexibility only with +a published contract — each violation is a build-time refusal with a +pointed error, never a downstream mystery): + +- **linux/amd64 required**; linux/arm64 is additionally used for + Apple-silicon local runs when the ref provides it, else amd64 + under emulation with a one-line performance warning; +- **glibc-based** — manylinux wheels and uv-managed interpreters + require it; musl/Alpine bases are refused; +- **apt-capable (Debian/Ubuntu family) iff `system-packages` is + nonempty** — otherwise refused, naming the two escapes (a + Debian-family base, or `Containerfile.extra`); +- **a POSIX shell at `/bin/sh`** (build stages run through it); +- **nothing else** — no Python, no pip, no uv (unlike Modal's + python-on-`$PATH` requirement, the base never supplies the + interpreter); `ENTRYPOINT`/`CMD`/`USER` are inert — lc invokes + with the entrypoint cleared, an explicit argv, and the invoking + uid (rootless). + +**Dependency edits are plain `uv add`, in both modes.** Escalating +never changes the dependency verbs: `uv add`/`uv remove`/`uv lock` +run **on the host, bare** — they are manifest-and-lock surgery, and +in containerized mode the environment is *derived* (the image), so +the next `lc materialize` picks the change up through the ordinary +rebuild path. (`lc run uv add …` is never the answer, and cannot +work: probes have no in-tree write scope, §4.) The three cases, +empirically grounded (uv 0.12.3, §13): + +- **Wheels or static-metadata sdists (the overwhelming majority)**: + plain `uv add` resolves and locks on the bare host with no system + layer needed. Its auto-sync side effect materializes a host + `.venv` that is **inert** in containerized mode — lc ignores it, + `lc status` notes it ("stray host .venv — inert in containerized + mode"), and `uv add --no-sync` avoids creating it. +- **A legacy sdist whose *metadata* build needs the system layer** + (no wheels, no static metadata — e.g. mysqlclient without + `mysql_config` on the host): `uv add` fails at resolution and + **rolls back cleanly** — `pyproject.toml` and `uv.lock` are left + untouched (verified). The documented remedy is uv's purpose-built + mechanism, `[tool.uv.dependency-metadata]`: declare the package's + metadata statically so resolution never builds it; the *actual* + build then happens inside the image, where the system layer + exists. (The setting flows into `uv.lock`, so identity is + covered by the lock bytes.) +- **A package needing the system layer only at build/install time** + (static metadata, e.g. rpy2): resolution and locking need no + build, so the lock is the deliverable and the host auto-sync's + fate is irrelevant — `uv add --no-sync` sidesteps the doomed host + install entirely, and the agent notes name it as the + containerized-mode habit. + +**Above the base, uv takes over — identically for every base.** The +layering is fixed by the generator, never user-ordered: (1) the apt +layer (`system-packages`); (2) the pinned uv binary +(engine-constant digest); (3) `uv python install` of the exact +`.python-version` interpreter into `/opt/python`; (4) +`uv sync --locked --exact --no-install-project --compile-bytecode` +into `/opt/venv` from `pyproject.toml` + `uv.lock` only; (5) the +`Containerfile.extra` stage, if declared — a stage `FROM` the +derived image, its content hashed into both `env_version` and the +tag, rendered by the generator; (6) the final ENV contract (§11 +step 6). The base contributes the OS layer and nothing else — it +can never supply the interpreter or a Python package, which is what +keeps environment identity uniform across bases. + +## 3. Identity + +**`env_version` = sha256(uv.lock bytes ‖ `.python-version` bytes ‖ +canonical install-settings JSON ‖ canonical `[tool.lightcone.image]` +JSON (base digest-ref, sorted `system-packages`) ‖ +`Containerfile.extra` content hash-or-null)** — computed by the +locked engine; sits inside `code_version`; also the mid-run gate's +baseline. Direct-mode projects hash empty system fields, so the +formula is one formula, not two. The install-settings set is v4's +closed, audited list (`default-groups`, `no-binary[-package]`, +`no-build[-package]`, `config-settings`, +`no-build-isolation[-package]`). When lc detects that the current +`env_version` differs from existing manifests, it prints the blast +radius — "environment changed: N materialized outputs are now +stale" — including at escalation time (§1). + +**Image identity (containerized only).** +**`tag = lc-env-`**. The rendered Containerfile embeds the +digest-pinned base (the generator's default — an engine constant — +or the project's declared `base`, §2), the digest-pinned uv binary, +the interpreter version, the apt list, and the `Containerfile.extra` +stage; the default-base and uv digests are generator constants +shipped with the locked engine, so new constants reach a project +only through an engine relock — tag and `env_version` move together; +a declared `base` moves them through the repo, like any environment +edit. Builds are incremental (tag hit ⇒ +no-op). **The build records the produced digest** (project-local +record + OCI label carrying `env_version`), and **execution is +digest-pinned**: the driver resolves tag → digest once at run start, +embeds the digest in every job command, and `run_rule` asserts it — +the image-side twin of the mid-run relock gate; a tag that resolves +to a different digest than the build record is a loud error, never a +silent substitution. **Code edits change no input to the tag** — the +image contains no project code, ever (G5). + +**Honest residue, stated:** the apt layer is name-pinned only — two +builds of one tag at different times can hold different system +package *versions*. The build records a `dpkg -l` snapshot and the +manifest stores its **sha256 (content hash), with the snapshot text +archived beside the build record** — so the attestation outlives +image garbage-collection. (Pinning via snapshot.debian.org is a §12 +hardening candidate.) This is attestation-grade system-layer +identity, accepted with eyes open in the decision record. + +**Lock scan**: refuse path/directory/editable dependencies except the +project's own package; report PyPI `sdist_built`; advisory on +non-default dependency groups. All v4 honest boundaries carry over +(raw-bytes over-invalidation; set-level-not-byte-level checks). + +**Manifest schema (normative field list — the single enumeration the +`SCHEMA_VERSION` bump implements and golden tests pin):** the v5-era +core (`schema_version`, `code_version`, `data_version`, +`env_version`, `recipe`, `decisions`, `input_versions`, `git_sha`, +`git_dirty`, `lc_version`, `host`) plus: `uv_version`; `platform` +(os-release, kernel, glibc, arch); `python_build`; `worker_runtime` +(`host` | `container`); `image` (`{tag, digest}` when a container +ran — project image or a static runtime image); `dpkg_snapshot_sha256` +(containerized); `sdist_built` (+ `cc --version` line when +nonempty); `env_snapshot` (locale, TZ, threading knobs); +`gpu_driver`; `hermeticity` (§7). **Scoping rule:** when +`worker_runtime: container`, every environment-describing field +(`platform`, `python_build`, `env_snapshot`, `gpu_driver`) is +captured *inside* the boundary — which is automatic in v6.1, since +`run_rule` itself executes there. `lc verify` surfaces dirty-tree and +unsandboxed outputs distinctly; pre-migration manifests report +`pre_migration`. + +## 4. The `lc` entrypoint + +Canonical, documented invocation on every venue: **bare +`lc `**, provided by the `uv tool install lightcone-cli` shim. +(`uv run lc ` is an accepted equivalent where the shim is not +installed — it cannot be the documented form, since it does not work +before a lock exists.) The shim is a launcher: (1) **discover** by +`astra.yaml` walk-up — uv's native walk-up discovery is never +trusted: every uv invocation carries `--project `; (2) +**detect the mode** (§1); (3) **scrub** the ambient `UV_*` namespace +(the closed v4 list); the values re-injected afterwards come from +**`site_registry.py`** (e.g. Perlmutter's `UV_CACHE_DIR` + +`UV_LINK_MODE`), with a matching ambient value honored as an +override — explicit flags beat ambient variables (§13), so the scrub +is defense-in-depth on lc's always-pass-explicit-flags posture; (4) +**converge**: direct mode — `uv sync --locked --exact +--compile-bytecode` where writable (captured output, prune notice); +containerized mode — resolve tag → digest against the build record; +**`lc materialize` builds a missing image** (printing "building +`lc-env-` — first run after an environment change; ~minutes"), +**`lc run` never builds** — it errors with the exact `lc build` +command; (5) **delegate by direct exec**: direct mode — exec +`/bin/lc` (`LC_DELEGATED=1`); containerized mode — `podman run` +into the digest-pinned image and exec `/opt/venv/bin/lc` there. The +delegation interface — argv passthrough + `LC_DELEGATED=1` — is +declared **minimal and frozen**: a tool-env launcher of any version +must be able to delegate to a project-locked engine of any age. +`status`/`verify` and pre-lock verbs run in the tool env. `lc build` +on a direct-mode project is an explanatory no-op ("direct mode — no +image to build; declare `[tool.lightcone.image]` to +containerize"). + +**`lc run CMD`** stays the thin probe verb and is mode-faithful +(G4): direct mode ≡ `uv run --locked --exact CMD` from the project +root, inside the §7 sandbox (`--no-sandbox` opts out); containerized +mode runs the same command inside the digest-pinned image under the +same policy. Bare `lc run` opens a shell in the recipe environment +and announces it ("opening a shell inside the recipe environment +(sandboxed)"). A probe has no output, so its read allowlist is the +union of all declared inputs, and its **write scope is the tmp +scope only** (§7) — never in-tree. **Rename guard** (v6 reassigns +`lc run` from pipeline execution to probing): a first argument that +matches a declared output id errors *before any exec* with "outputs +are materialized, not run — did you mean: `lc materialize +best_fit`?". + +**`lc status`** stays manifest-driven — the invariant is now +**offline and local-only** (it additionally reads `pyproject.toml` +and the local image store, never the network) — and gains three +header lines answering the questions nothing else surfaces: + +``` +mode: containerized (3 system packages) # or: direct +image: lc-env-9f2c81d44a1b03e7 — built (digest sha256:…) # or: needs build +sandbox: landlock (fs: declared, network: unenforced) # this host +``` + +The denial message and the podman-missing error both point here. + +## 5. Venues and placement + +| Venue | direct | containerized | +|---|---|---| +| Laptop (Linux/WSL2/macOS) | `.venv` in-tree; nothing to configure | + rootless **podman** (docker accepted); the full stack runs in the image. macOS runs it in `podman machine` — a one-time Linux VM setup, announced in the first denial (§7); recipes execute linux builds resolved from the same universal `uv.lock`; no GPU inside the VM, stated. **Mount-source preflight**: every mount source (project root, declared inputs, scratch) must lie under the machine's shared directories (`podman machine inspect`); a source outside them is a *refusal* naming `podman machine set --volume` — never a silently empty mount | +| Perlmutter (2–4 nodes) | Project on **CFS, not `$HOME`**; cache placement supplied by the site registry (§4); CFS writable from compute ⇒ sync works mid-salloc; `scratch.py` redirects + Lustre run lock kept; `_abort_on_perlmutter_login` kept, materialize-scoped. The recipe wrap uses the **static runtime image**: materialize preflight checks it and, from a compute node, errors with the exact login-node command (`lc … --pull-runtime-image`, wrapping `podman-hpc pull` + `migrate`) | **spike-gated** (§11): full stack via `srun podman-hpc run --net=host … dask worker` from the project image (built/migrated once on a login node; preflight prints the exact command). `--net=host` is required for dask reachability and is what the network posture honestly records. GPU rules hang on the `--gpu`/`--net` CDI spike; until it passes, containerized GPU-on-Perlmutter refuses with the BYO/direct alternatives named | +| Hub / GKE (few workers) | Static deployment-managed runtime image (by digest); project env + interpreter + code ride same-path NFS home; lc passes the env location as **`LC_PROJECT_ENV`** through the standard `environment` cluster option; the shim execs `/bin/…` and **fails loudly** when unset/absent (never a PATH fallback — §13); deployment image recorded as attestation. G4's documented exception lives here: notebook-pod probes share the lock but not the workers' OS layer; `worker_runtime` keeps it inspectable | worker pods run the **project image** (full stack: the dask worker, the child snakemake, and the recipe all in-image); code via the NFS workdir mount; Cloud Build renders from a three-file context (rendered Containerfile, `pyproject.toml`, `uv.lock`). **Deployment contract**: the notebook image for a containerized project is derived `FROM` the project image (one generated layer adding Jupyter), so the driver runs the same OS layer and `/opt/venv` — restoring G4 exactness on this venue; where the deployment cannot provide it, containerized-mode materialize refuses at preflight, naming the contract item and the BYO/direct alternatives | +| External scheduler | Caller-owned; no-write posture + fingerprint gate as v4 | same; image availability and digest pinning are the caller's contract, verified by the connect probe | + +## 6. `lc materialize` and the fabric + +Flow unchanged (Snakefile → run-scoped cluster → rules as dask tasks +→ manifests). The integrity core: + +- **Converge once, then never write to the environment.** Direct + mode: driver preflight `uv sync --locked --exact` before cluster + start; workers run `uv run --no-sync` with the offline overlay + (`UV_OFFLINE=1`, `UV_PYTHON_DOWNLOADS=never`) and scrub-list unsets + in every job command. Containerized mode: the *build* is the + convergence; at run time nothing syncs anywhere — the driver + resolves tag → digest (building via the §4 rule if absent) and + every process runs from the immutable image env. +- **The worker sequence** — inside `run_rule` (which in containerized + mode itself executes inside the image): + 1. **pre-gate** — re-hash the `env_version` inputs (mounted project + tree) against the run-start value in the job command; + 2. **env check** — direct: assert the env prefix exists and + `uv sync --locked --exact --check` (true no-write env-vs-lock + verification); containerized: **assert the running image's + digest equals the driver-resolved digest** in the job command; + 3. **exec the recipe through the boundary** — direct: the sandbox + shim (§7); containerized: the same shim, applying in-container + Landlock (§7); + 4. **post-recipe re-gate** before `write_manifest`. +- **Mid-run relock gate** (double) and **driver-side git capture** — + verbatim from v4. +- **Connect-time probe** in `_connect_client` (all branches, + self-contained closure): workdir mount + engine version + ( + containerized) image digest; `--allow-unverified-cluster` on the + external branch. + +## 7. Hermeticity enforcement + +**The guarantee (G6):** a recipe cannot use executables or files +outside the declared set wherever a mechanism exists, and every +manifest records exactly what ran. Research: +[hermeticity-enforcement.md](hermeticity-enforcement.md). + +**The default filesystem policy** (one policy, both modes): + +- **write**: own `results///` output dir + scratch + `/tmp` + + `/dev/shm` (+ `/dev/null`) — protecting sibling outputs, + manifests, and `astra.yaml` from a misbehaving recipe. The + per-output escalation for recipes that legitimately write + intermediates in the tree is **ASTRA-declared only** — the + `sandbox: writable-project: true` key on the output (no CLI flag: + a behavior-changing choice must live in the repo, reproducible + from files a year later); such outputs record + `hermeticity.fs: project-rw`. +- **read**: the project tree; declared ASTRA inputs outside it; the + OS baseline — `/usr`, `/lib`, `/etc`, `/proc`, `/sys`, + `/dev/urandom`, locale/SSL data. +- **HOME and XDG, normative in both modes**: the boundary sets + `HOME`, `XDG_CONFIG_HOME`, `XDG_CACHE_HOME`, `XDG_DATA_HOME`, and + `MPLCONFIGDIR` to a **fresh per-recipe directory under the + writable tmp scope** (the Bazel/nix move). The real `$HOME` is + neither readable nor writable: matplotlib/astropy/R work on first + import, and the dotfile-steering channel stays closed — an + implementer must never "fix" a HOME failure by mounting `$HOME` + RO. +- **execute — two tiers (direct mode)**: the `.venv` binaries + **plus an enumerated, versioned utility set** (`sh`, `bash`, + `env`, coreutils, `grep`/`sed`/`awk`, `tar`/`gzip`) **plus the + realpath'd ELF loader(s)** (`/lib64/ld-linux-*`, musl's loader + where present) — Landlock checks EXECUTE on the interpreter's + open, so without the loader every dynamically linked binary, + python and bash included, fails EACCES; shared libraries need only + the read baseline. The allowlist is a maintained policy surface, + recorded in the manifest by version, with a policy unit test that + execs a dynamically linked binary. In **containerized mode the + image contents are the exec set** — everything present was + declared. +- *(Amended at implementation, 2026-08:)* the boundary additionally + sets **`PYTHONPYCACHEPREFIX`** to the per-recipe tmp scope, so + in-tree bytecode caches redirect there — the env-RO `__pycache__` + consequence this bullet originally accepted is eliminated without + widening any grant. (The env itself remains pre-compiled: + `--compile-bytecode` in both the direct-mode converge and the + image build.) + +**Boundary placement — direct mode: the exec-shim.** The sandbox +wraps **the recipe, not uv**: step 3's command is + +``` +uv run --no-sync … -- python -m lightcone._sandbox_exec -- bash -c '' +``` + +uv, its config, and its caches are trusted plumbing outside the +boundary (protected by the no-write worker posture). On Linux, +`run_rule` builds the Landlock **ruleset FD before fork** and passes +it down (`pass_fds` + env); the shim performs exactly two raw calls — +`prctl(PR_SET_NO_NEW_PRIVS)` + `landlock_restrict_self(fd)` — then +`os.execv`'s bash. No `preexec_fn`. **Open verification item +(spike): FD survival through uv's spawn/exec chain** — a Landlock FD +cannot be reopened; if uv drops non-stdio FDs, the documented +fallback is step 3 exec'ing the resolved interpreter directly, +bypassing the `uv run` hop. On macOS the shim execs +`sandbox-exec -f /bin/bash -c ''` — the +profile generated with **realpath'd** paths including `TMPDIR`, plus +`(allow ipc-posix-shm*)`/`(allow ipc-posix-sem*)` and localhost +`network-outbound`. `sandbox-exec` cannot nest — a recipe invoking it +fails by design. The LocalCluster worker needs no exclusion: only +step 3 (the recipe exec, §6) is wrapped — this is why the design +composes with the fabric. + +**Boundary placement — containerized mode: mounts bound the world, +in-container Landlock scopes the recipe.** The engine container +mounts: project tree RW at its identical absolute path (the engine +writes results and manifests); each declared external input RO; +scratch and a tmpfs `/tmp` RW; nothing else from the host. The env +is baked (`/opt/venv`) — no env mounts. Within the container, step +3 applies the **same Landlock exec-shim** per recipe (the container +is Linux on every host OS, macOS included), scoping each recipe to +project-RO + own-output-RW exactly as direct mode does. In-container +Landlock requires kernel ≥5.13 and a seccomp profile admitting the +`landlock_*` syscalls (default in current podman/GKE profiles; +probe-checked per job, never assumed). With it, file scope is +`declared` (mechanism `podman+landlock`); without it, the honest +weaker value `project-rw` (mechanism `podman`). + +**The mechanism matrix:** + +| Venue × mode | Mechanism | File scope | Network | +|---|---|---|---| +| Linux laptop / WSL2, direct | **Landlock** (vendored ctypes; floor ABI 1 / kernel 5.13; probe per job, ABI recorded) | `declared` | `unenforced` (ABI ≤ 3 has no network control; ABI 4 cannot express a loopback carve-out — recorded, not pretended) | +| macOS laptop, direct | **Seatbelt** generated SBPL (capability-checked per OS release; CI smoke) | `declared` | `denied` (non-loopback; localhost allowed) | +| Laptop, containerized | **podman+landlock** (probe-checked; `podman` without) | `declared` (`project-rw` without in-container Landlock) | `denied` — engine container runs `--net=none`, loopback intact (in-recipe LocalCluster/torch workers keep working) | +| Perlmutter, direct | **podman-hpc recipe wrap** with the static runtime image (step 3 only — never the dask worker). Mounts: project RO, own output dir RW, realpath'd uv interpreter dir RO, declared inputs RO, `$SCRATCH` RW — the normative policy, honestly expressed; `$PWD`-RW is **never** stamped `declared`. **Landlock replaces this wrap if the SLE-15 + Lustre spike passes** (deletion path pre-declared: the wrap, its mount code, and the static image's Perlmutter role all go) | `declared` | `denied` via `--net=none` — *provisional on the `--gpu`/`--net` spike; fallback posture if incompatible: GPU rules run unwrapped and record `mechanism: none` rather than pretend* | +| Perlmutter, containerized (spike-gated) | **podman-hpc full-stack** (project image; workers under `--net=host`) + in-container Landlock per recipe | `declared` (`project-rw` without) | `allowed` — `--net=host` is what dask requires, recorded honestly | +| Hub, direct | **pod** (static image) — the pod bounds the OS layer but worker pods mount all of NFS home; declared-file discipline comes from **Landlock inside the pod** (chart item: seccomp `RuntimeDefault` + `landlock_*` allowlisted) or not at all | `os-only` (`pod+landlock` ⇒ `declared`) | `allowed` unless NetworkPolicy — workers record `allowed` unconditionally (lc cannot observe NetworkPolicy) | +| Hub, containerized | **pod** (project image, full stack) + in-pod Landlock as above | `os-only` (`pod+landlock` ⇒ `declared`) | `allowed` (same rule) | +| Native Windows | none | `open`; WSL2 users get Landlock | — | + +**Network enum, normative mapping:** `denied` means non-loopback +egress is blocked (loopback always intact — this is the spec's +meaning of denied, Seatbelt and `--net=none` alike); `allowed` means +lc applied no restriction (recorded whenever the mechanism's flags +did not deny — the matrix row is documentation, the *flags actually +applied* are what the manifest records); `unenforced` means the +mechanism cannot express a useful deny (Landlock). Per-output +network declarations are **deferred to §12** — v1 remedies for a +network-needing recipe are declaring the download as an input, or a +recorded `--no-sandbox` run. + +**Manifest field:** +`hermeticity: {mechanism: landlock|seatbelt|podman|podman+landlock|podman-hpc|pod|pod+landlock|none, +fs: declared|project-rw|os-only|open, network: denied|allowed|unenforced, +landlock_abi?, exec_allowlist_version?}`. + +**Probe, strictness, and never-silent — worker-side, per job**: the +capability probe runs in `run_rule` (the driver's kernel is not the +worker's) and populates the manifest there. When the probe lands +below the venue's expected level (`mechanism: none`, or `project-rw` +where `declared` was expected), lc records it **and prints one +console line** — a user must never finish a materialize believing +they were sandboxed when they weren't. `--require-sandbox` is +enforced there too: bare form requires `mechanism ≠ none`; +`--require-sandbox=declared-fs` additionally requires +`fs: declared`. **Maintenance note:** the podman-hpc and GKE-seccomp +behaviors were validated by one-time spikes, but NERSC and GKE +upgrade underneath them — the venue rows are a maintained surface +like the exec allowlist; revalidate on site changes (a periodic +canary materialize per venue is the cheap form). + +**The denial UX — the design's primary UI (mandatory).** The parent +re-stats on `EACCES`/`EXDEV`/sandbox-`ENOENT` and classifies by +best-guess heuristic (exec bit / bin-dir path ⇒ tool; otherwise ⇒ +data file), ordering remedies accordingly — both always shown, each +with a copy-pasteable fix: + +``` +blocked by lc sandbox: cannot execute /Library/TeX/texbin/latex — +not part of the declared environment. + + if this is a tool the recipe needs, declare it in the system layer: + [tool.lightcone.image] + system-packages = ["texlive-latex-base"] + (apt package names — unsure? try: apt-cache search ) + note: this containerizes the project — podman required (macOS: + one-time `podman machine` VM setup, ~minutes) — and re-stages + all materialized outputs. + + if this is a data file, declare it as an input in astra.yaml: + outputs: + report: + inputs: + - path: /Library/TeX/texbin/latex + + diagnostics: lc run --sandbox-debug (shell inside the sandbox) · + lc run --no-sandbox (recorded as unsandboxed) · lc status +``` + +`--no-sandbox`/`--sandbox-debug` live in the subdued diagnostics +trailer, never as peer remedies. The tool → apt-name hint table is +**capped and versioned** (a dozen high-frequency tools: latex, R, +julia, convert, pdftoppm, …), falling back to the generic +`apt-cache search` line — never an open-ended mapping. A wrong +package name surfaces as lc parsing apt's +`E: Unable to locate package X` inside the build into "no apt +package named `X` — search with `apt-cache search`", never a raw +build log. **And on every nonzero sandboxed exit** — including +recipes that swallow the PermissionError — lc appends a fixed +one-line trailer: "this recipe ran under the lc sandbox (landlock) — +if the failure looks like a permissions/missing-file error, try +`lc run --sandbox-debug`". Threat model stated (declared-dependency +discipline against *accidental* leakage — metadata visibility, +interpreter-reads-script, memfd-exec are named adversarial-only +gaps); realpath every policy path. + +## 8. Deleted ledger + +v4's deletions stand (placement tiers, markers/leases/`lc env` +verbs, `lc doctor`, fingerprint ladder, projection document). v6/v6.1 +amendments: + +- **The pixi substrate (v5) is deleted** — rejected; decision record + and re-add triggers: + [substrate-default-tradeoffs.md](substrate-default-tradeoffs.md). + The §13 pixi empirical evidence block is the sole authoritative + copy and is retained by recorded decision. +- **The recipe-only container wrap (v6.0) is deleted** — replaced by + full-stack-in-image after the review's host-sync deadlock finding: + a hybrid that syncs the lock on the bare host cannot support + lock-level system dependencies, which are the hatch's own §2 + examples. +- **`lc build` / `container.py` / `cloudbuild.py` return, reworked** + — the tag+digest functions, the generated code-free Containerfile, + and a three-file Cloud Build context replace the v3-era + build-input hashing and project-tree staging. +- **Two flags deleted before birth**: `--accept-containerfile-loss` + (v6.0 migration constant) — `lc init` instead *refuses* on an + authored Containerfile ("v6 generates images from the lock — + delete or rename it, then re-run `lc init`"; the user's own file + operation is stronger consent than a flag); and + `--sandbox-writable-project` — the ASTRA-declared per-output key + is the single spelling (§7). +- **Per-output `network:` declarations** — dropped from the + normative surface and the ASTRA migration step; returned to §12 + (the motivation stands: `--no-sandbox` being all-or-nothing is why + the design deserves finishing). +- The **BYO digest-pinned per-output container** remains the + schema's escape for truly arbitrary cases. +- Diagnostics re-add candidates: doctor-style checks, RECORD-hash env + audit, trace attestation, image GC. + +## 9. Fabric: dask, seam explicit, Ray escape hatch + +Unchanged from v4 (keep dask; seam = `cluster_for_run` + executor; +Ray `working_dir`+uv is the shared-FS-less escape; re-evaluate on the +named triggers; dask-gateway risk + hub contract carried). + +## 10. Alternatives + +- **Dual-substrate uv + pixi (v5)** — rejected; see + [substrate-default-tradeoffs.md](substrate-default-tradeoffs.md). +- **pixi-only** — rejected for the default on adoption grounds. +- **Recipe-only container wrap (v6.0)** — rejected by review: the + host-sync deadlock (§8). Full-stack-in-image costs the podman-hpc + worker-launch machinery on Perlmutter (spike-gated) and buys + lock-level system deps, engine coherence, in-container Landlock on + macOS, and the disappearance of the dual host/image environment. +- **Container-canonical (v1)** / always-container — rejected; + Landlock/Seatbelt deliver direct-mode discipline with zero + install, and the majority path never needs an image. +- **uv-only without the hatch (v4-pure + BYO)** — rejected: leaves + G6's denial without a first-class remedy. +- bubblewrap — future upgrade tier. Trace-based hermeticity — future + attestation. Snakemake `--sdm`, venv+activation, Ray-now — + rejected as before. + +## 11. Migration + +Stage 1 (unchanged, first): rename `lc run`→`lc materialize`; the +new `lc run` probe verb **with the rename guard** (§4); hint; docs; +tests. + +Stage 2 — constant: `LC_DELEGATED=1`. Steps: + +1. **Environment layer**: `env_version`; lock scan; the §3 manifest + field list; `SCHEMA_VERSION` bump; digest build-record; generator + ignores legacy `container:`; golden fingerprint fixtures (direct + and containerized). +2. `lc init` (uv scaffold; authored-Containerfile refusal; AGENTS + stanza) + launcher (discover → mode-detect → scrub with + site-registry re-injection → converge/resolve → exec; frozen + delegation interface; `lc status` header lines; packaged× + containerized refusal). +3. **Static runtime image** (deployment repo): slim base + uv + + exec-shim, by digest — the hub's direct-mode substrate and + Perlmutter's direct-mode wrap image; direct-mode Perlmutter + preflight with the exact login-node command. +4. Fabric: no-write job command + worker sequence (pre-gate, env + check/digest assert, boundary exec, post-gate); offline overlay + + scrub unsets; driver git capture; connect probe (+digest); + login-guard scoping; error-string sweep. +5. **Sandbox layer** (direct): vendored Landlock ctypes + policy + builder (incl. **ELF loader tier** and **HOME/XDG contract**) + + the `lightcone._sandbox_exec` shim (FD-inheritance spike first; + direct-interpreter-exec fallback specced); Seatbelt generator + + macOS CI smoke; podman-hpc direct wrap **with the honest mount + set** (spike-contingent, deletion path pre-declared); + probe/attestation + downgrade console line; the §7 denial UX + (heuristic ordering, capped hint table, apt-error parsing, VM + cost notice, failure trailer); `--require-sandbox` / + `--no-sandbox` / `--sandbox-debug`. +6. **Container hatch (laptop + hub)**: `[tool.lightcone.image]` + + `Containerfile.extra` parsing and full rendering, with the §2 + base-contract refusals (digestless base, musl base, apt-less base + with `system-packages`, arm64 emulation warning); the + Containerfile generator — apt layer before `uv sync`; **offline + ENV (`UV_OFFLINE=1` etc.) emitted only in the final stage**, so + the build's own sync layer keeps network (pinned by golden + tests); `--compile-bytecode`; dpkg snapshot + content hash; + world-readable; tag + digest record; builders — podman (canonical + local), Cloud Build (three-file context); `lc build`; the + full-stack run wrapper (engine-container mount set, `--net=none`, + in-container Landlock probe); macOS `podman machine` preflight + incl. the **mount-source share check**. +7. **Perlmutter containerized** (spike-gated): podman-hpc full-stack + launch (`--net=host` workers, login-node build/migrate preflight, + GPU-CDI gate with refusal until passed). +8. Hub plumbing: `LC_PROJECT_ENV` (direct); project-image pods + + the notebook-`FROM`-project-image deployment contract with its + preflight refusal (containerized); attestation-only image + recording. +9. ASTRA: environment declaration = manifest+lock+system layer; + `container:` derived/optional (BYO per-output container stays); + the `sandbox: writable-project` output key; WRROC (the RO-Crate + provenance archive format) archives the lockfile. +10. Eval re-baseline (sandbox on; one containerized eval task). +11. Cleanup (`_unpack_result`; superseded docs). + +Tests land with their steps, plus: Landlock policy units — allow, +deny, crisp-message, **dynamically-linked-exec**, and the two +denial-fallback fixtures (a recipe that swallows PermissionError; a +rewrapped error that defeats the re-stat classifier — the trailer +must fire in both); Seatbelt smoke on macOS CI; +generated-Containerfile golden tests (incl. ENV ordering) + a +`podman build` smoke; mount-set units; CliRunner coverage for both +modes. + +**Spike (gates the steps that name it):** the v4 Perlmutter items; +**Landlock ruleset-FD survival through `uv run`'s spawn/exec chain** +(gates step 5's shim shape); Landlock SLE-15 + Lustre (if it passes, +replaces the direct-mode podman-hpc wrap); podman-hpc recipe-wrap +smoke (mount set, `-e` pass-through, `--net=none`, the +`--gpu`/`--net` interaction with its stated fallback); podman-hpc +full-stack smoke (worker launch, `--net=host`, GPU CDI — gates step +7); rootless-podman laptop smoke (uid mapping, mount perf, +`--net=none`, **landlock syscalls under the default seccomp +profile**); macOS `podman machine` (file-sharing perf, tmpfs, exit +codes, share coverage); hub-pod seccomp (`landlock_*` under GKE +defaults). + +## 12. Open questions + +1. **The project survey** — which existing projects hold + dependencies uv cannot source, and on which venues their users + sit. Sizes the hatch population, gates eval depth, and remains + the standing pixi re-add datum. +2. **apt pinning** — snapshot.debian.org (or distroless/wolfi bases) + to harden the name-pin residue. +3. **Per-output `network:` declaration** — deferred from the v6.0 + surface; `--no-sandbox` being all-or-nothing is the motivation to + finish the design (ASTRA schema; default deny where expressible). +4. **Per-output environments** — deferred; one-env-per-project is + what this would relax deliberately. +5. **Landlock-on-Perlmutter** — spike-gated replacement of the + direct-mode podman-hpc wrap. +6. **Image lifecycle** — `lc`-side image GC and an accumulation + policy for Perlmutter's migrated images (today: podman's own + `image prune`, documented). +7. **bubblewrap upgrade tier**; **wheel variants (PEP 825)** — + every adopted variant shrinks the hatch population; **fabric + re-evaluation clock**; **scale re-add trigger** — as v4. + +## 13. Evidence appendix + +Carried from v4 (fabric ground truth; uv 0.12.3 empirical set incl. +the `UV_OFFLINE` cache-write finding, `--exact` additive-sync +finding, read-only-venv warm-run verification, **explicit flags beat +ambient `UV_*` variables**, and `uv run`'s silent PATH fallback + +`--no-sync` disabling the locked check — the facts behind the hub +shim's fail-loud contract). **`uv add` semantics pass (uv 0.12.3, +2026-08-17, grounds §2's dependency-edit rules)**: on a +resolution-phase build failure (legacy sdist without static +metadata, build tool absent from PATH, cold cache) `uv add` **rolls +back** — `pyproject.toml` and `uv.lock` left untouched — and uv's +own hint names `--frozen`; on success, the auto-sync creates the +host `.venv` as a side effect (`--no-sync` avoids it); a warm uv +cache can mask the build failure entirely (the build is skipped on +a cache hit). +[hermeticity-enforcement.md](hermeticity-enforcement.md) — Landlock +ABI/distro map, Seatbelt precedents, bubblewrap gate, tracing. +Review-derived design facts (v5.1 + the v6 review record): the +two-tier exec baseline; the exec-shim boundary; Landlock +additive-rights expressiveness; **Landlock EXECUTE applies to the +ELF loader open** (the §7 loader tier); hub pod = `os-only` without +in-pod Landlock; Landlock ABI ≤ 3 has no network control and ABI 4 +cannot express a loopback carve-out. **Open item, spike-tracked: +Landlock ruleset-FD inheritance through `uv run`** (§7/§11 — no +verified entry exists yet; the fallback is specced). Prior art for +the ladder: Metaflow `--environment=uv` (ships +`pyproject.toml`+`uv.lock`, syncs frozen, execs via +`uv run --no-sync`; no pixi backend exists — verified against source +at `2fb3c91`), Flyte/Union ImageSpec, Modal +(`Image.apt_install(...).uv_sync()`), ClearML `uv sync --locked`. +**Retained decision-record evidence — pixi 0.76.2 empirical pass** +(grounds the §8 rejection and any future re-add): +`--locked`/`--frozen` mirror uv; `--no-install` cold-env silent +host-PATH fallback; `pixi lock --check` writes on drift +(`--dry-run --check` is the safe form); env self-containment + +`env -i` direct exec; URL+sha256 PyPI locking and hash-free path-dep +entries; the closed `PIXI_*`/`CONDA_OVERRIDE_*` steering list; +interpreter build + channels/indexes inside `pixi.lock`; lock header +`version: 7`. **uv→pixi migration pass** (pixi 0.76.2 · uv 0.12.3): +`[project.dependencies]` carry over verbatim; same-day migration +resolves byte-identical PyPI artifacts; pixi ignores +`.python-version`; no uv.lock import exists; bare `pixi init` on an +existing pyproject prompts interactively, injects an editable +self-dependency, and defaults to a single platform. Review record: +[execution-environment-v6-review.md](execution-environment-v6-review.md). diff --git a/docs/design/hermeticity-enforcement.md b/docs/design/hermeticity-enforcement.md new file mode 100644 index 00000000..dcdd1973 --- /dev/null +++ b/docs/design/hermeticity-enforcement.md @@ -0,0 +1,315 @@ +# Findings: hermeticity enforcement without a container stack + +- **Status:** research findings — evidence base for a future hermeticity + revision of [execution-environment.md](execution-environment.md) + (not yet normative; nothing here is implemented) +- **Date:** 2026-08-15 +- **Method:** 4 web-research agents (syscall-trace capture / ReproZip; + Landlock; bubblewrap; macOS Seatbelt), each verifying against primary + sources (kernel docs, man pages, project repos, shipped + implementations); plus the prior fabric/venue analysis in the spec. +- **The requirement (owner-stated):** it must be *mechanically + impossible* for an output to be materialized using tools or files + outside the declared environment without that fact being caught — + the property containerization provided by brute force — **without** + requiring a container stack on laptops, and ideally packaged inside + `lc` so it is 100% transparent to the user. + +## TL;DR + +**Landlock (Linux) + Seatbelt (macOS) deliver the "can't use stuff +outside the environment" guarantee natively, unprivileged, with +nothing to install — and both are battle-tested in exactly this +embedded-in-a-CLI role** (OpenAI Codex ships Landlock; Anthropic's +sandbox-runtime/Claude Code and Codex both ship Seatbelt; Bazel, +Chrome, and Arch's pacman are further production users). Bubblewrap is +a stronger isolation model but is blocked out-of-the-box on stock +Ubuntu 24.04/WSL2-Ubuntu for a wheel-shipped binary — usable only as +an opportunistic upgrade. Syscall tracing (ReproZip-style) remains +valuable as *attestation*, but with enforcement this widely available +it demotes from primary mechanism to optional evidence. The proposed +shape is a per-output **hermeticity ladder** recorded in the manifest: +`enforced` / `traced-clean` / `open` — never silent. + +## 1. What "capture the entire environment" actually decomposes into + +The container's guarantee is really two separable properties: + +- **Prevention**: a recipe *cannot* use anything outside the declared + set — undeclared tools/files don't exist in its world, so + irreproducibility is caught at materialization time as a loud + failure, exactly when the agent introduces it. +- **Detection**: if a recipe *did* touch something outside the + declared set, that fact is recorded and `lc verify` refuses to call + the output reproducible. + +Leakage channels, in order of practical frequency: PATH executables +(host `latex`, `module`-loaded tools, `/usr/local/bin` strays); +Python-level leakage (`PYTHONPATH`, stray user-site packages); +filesystem at large (absolute-path invocations, undeclared data files, +dlopened host libraries); network fetches mid-recipe (irreproducible +inputs); ambient env vars. Note the honest baseline: even full +containers never closed kernel, GPU driver, or CPU-dispatch channels — +those remain attestation under every mechanism. + +## 2. Landlock (Linux) — the transparent default + +A kernel LSM syscall API (mainline since **5.13**, June 2021): +a process self-restricts with an allowlist of path-scoped access +rights before exec'ing the recipe; restrictions are inherited by the +entire process tree and can never be removed, only tightened. + +**Why it uniquely meets the transparency bar:** +- **Zero binaries, zero privileges, zero setup.** Applied via three + syscalls (+`prctl(PR_SET_NO_NEW_PRIVS)`) — ~100 lines of stdlib + ctypes in a `subprocess.Popen(preexec_fn=…)` hook (child is + single-threaded there, so per-thread semantics are moot). Pure + Python; ships inside `lc` itself. PyPI bindings exist + (`landlock` — ctypes/MIT; `py-landlock` — covers net+scoping) but + vendoring the ~100 lines is the low-dependency path. +- **Works where bubblewrap doesn't**: stock Ubuntu 24.04, WSL2, and + (kernel-permitting) inside containers — it needs no user + namespaces and no AppArmor blessing. +- **The enforcement fits the need exactly**: deny + `LANDLOCK_ACCESS_FS_EXECUTE` outside {venv, uv-managed interpreter, + minimal OS baseline} ⇒ `subprocess.run("latex")` on an undeclared + tool fails instantly with `PermissionError`; deny reads outside + {project, venv, baseline} ⇒ undeclared data files are caught too — + which quietly resurrects the old `$PWD`-mount path discipline. +- **Overhead ≈ zero** at workload level: in-kernel checks at + open/exec time only — no ptrace context switches (2025 kernel work + moved worst cases toward O(1) per open). + +**ABI/kernel availability (verified):** ABI 1 (5.13, full R/W/X file +rights) → ABI 2 (5.19, REFER) → ABI 3 (6.2, TRUNCATE) → ABI 4 (6.7, +TCP bind/connect) → ABI 5 (6.10, IOCTL_DEV) → ABI 6 (6.12, signal / +abstract-socket scoping) → ABI 7 (6.15, **audit of denials**) → ABI 8 +(TSYNC; release carrying it to be re-verified). Distro reality: +Ubuntu 22.04 = ABI 1 (HWE 6.8 = 4); Ubuntu 24.04 = ABI 4+; Debian 12 += 2, Debian 13 = 6; Fedora ≥ 7; Arch 7–8 (pacman 7 itself now uses +Landlock); **WSL2 (msft 6.6 kernel) = ABI 3**. **Floor: ABI 1** — +anything higher would exclude 5.14-based HPC kernels. + +**Limits, stated honestly:** +- Metadata is visible (`stat`/`access`/`chdir` are not restrictable) — + files can be *seen*, not opened. Fine for fail-loudly-on-use; not + an information-hiding boundary. +- Not adversarial-proof: memfd-exec (`LANDLOCK_SCOPE_MEMFD_EXEC` is + still an RFC), interpreter-reads-script (EXECUTE gates `execve`, + not interpretation), fd smuggling. Irrelevant to the accidental- + leakage threat model; must be stated in the spec. +- Docker's default seccomp profile historically does not allowlist + the `landlock_*` syscalls → probe at runtime inside pods rather + than assume (custom seccomp profile is the hub-chart fix). +- Denials on ABI <7 surface only as EACCES/EXDEV in the recipe (the + 6.15 audit stream needs root to read); on ABI 1, cross-directory + rename/link out of allowed trees is denied wholesale (no REFER). +- NFS/Lustre: hooks VFS path resolution, filesystem-agnostic in + principle; **no field reports either way — Perlmutter smoke test + required.** SLE-15's `CONFIG_LSM` inclusion of landlock is + **unverified** (SLES 16 confirms it) — same probe. + +**Precedent:** OpenAI Codex CLI's Linux sandbox uses the kernel +author's own `rust-landlock` crate (`ABI::V5`, best-effort, full-FS +read + write-allowlist, network cut via a separate seccomp filter); +its filed issues are a free lessons-learned list — most importantly +the **silent-best-effort trap**: best-effort setup "succeeding" on a +kernel without Landlock means running unsandboxed without knowing. +`lc` must probe, record the effective ABI in every manifest, and +offer a strict mode that refuses to run unenforced. + +## 3. Bubblewrap (Linux) — stronger model, gated availability + +`bwrap` (containers/bubblewrap, v0.11.2, Apr 2026; Flatpak's sandbox, +also under Steam) builds an **empty mount namespace** — nothing exists +inside except what is explicitly `--ro-bind`/`--bind`-ed — plus +`--unshare-net` for total network deny, `--die-with-parent`, +`--clearenv`. Policy-as-mount-table: undeclared paths yield `ENOENT` +("doesn't exist" — arguably cleaner UX than `EACCES`), read-only binds +yield `EROFS`. + +**The availability wall (verified in detail):** unprivileged user +namespaces are AppArmor-gated **by binary path** on Ubuntu 23.10+ / +24.04 LTS (and therefore WSL2-Ubuntu): the apt-installed +`/usr/bin/bwrap` is whitelisted only on 25.04+ (the 24.04 profile was +shipped and then reverted), and a **wheel-shipped bwrap matches no +profile and is blocked** — the exact wall Codex, VS Code, melange, and +Anthropic's sandbox-runtime all document, all resolved by "prefer +system bwrap from PATH, else print the one-sudo-command remediation". +Docker/K8s default seccomp also blocks the required `clone` flags, so +bwrap inside pods is unreliable. Debian/Fedora/Arch/openSUSE work +untouched. Rough estimate: only ~40–60 % of Linux laptops run a +wheel-shipped bwrap with zero admin action today. + +**Packaging is otherwise trivial**: static-musl builds are a known +recipe, ~100–300 KiB per arch, LGPL-as-aggregated-subprocess is the +easy license case, and Codex already ships a bundled `bwrap` +system-first. Runtime overhead: milliseconds of setup, native speed +after. + +**Verdict: opportunistic upgrade, never the requirement.** Where +usable it adds empty-world isolation, PID isolation, and clean +network unshare on top of Landlock; where gated, Landlock carries the +guarantee alone. + +## 4. Seatbelt / `sandbox-exec` (macOS) — deprecated in name only + +- Deprecated in the man page since ~2012; **no removal timeline ever + published**, still functional through macOS 26, and it cannot + realistically be removed: Seatbelt is the substrate of Apple's own + App Sandbox and of the system profiles confining Apple's daemons. +- **Production users of exactly our pattern**: Anthropic + sandbox-runtime (Claude Code's `/sandbox`) — generated SBPL + profiles via the `sandbox-exec` binary, writes deny-by-default, + network via localhost-proxy allowlisting; OpenAI Codex — + parameterized deny-by-default profile, network omitted unless + opted in; Bazel's `darwin-sandbox`; Chrome; Homebrew; SwiftPM. +- **Inheritance is the key win**: the sandbox applies to the whole + descendant tree; children cannot shed it. (Corollary: no nesting — + a recipe that itself calls sandbox-exec fails; Bazel's fallback + exists for this.) +- Practical profile: `(deny default)` + project dir RW (via + **realpath'd** `subpath` — `/tmp`→`/private/tmp` symlinks are the + classic silent miss), venv + interpreter RX, `/System`,`/usr/lib`, + dyld caches, `/dev/{null,urandom}`, locale/ssl baseline RO, + `(deny network-outbound)` by default (selective host allowlisting + is not expressible in SBPL — binary allow/deny per recipe is the + robust policy, as Codex chose). +- Risk posture: version-gated capability check + graceful fallback to + trace/warn + documented opt-out; treat the generated profile as + maintained code with a macOS CI smoke test (per-release path drift + is the realistic breakage, not removal). Debuggability is the weak + spot (`(trace)` was removed; `log stream` shows violations). +- Alternatives rejected: Endpoint Security (Apple-granted entitlement + + bundle — nonstarter for a pip/uv CLI); App Sandbox (wrong model); + Apple Containerization/`container` 1.0 (2026: per-step Linux VMs, + macOS 26 + Apple Silicon only — changes the substrate rather than + confining the native env; watch as a future opt-in hermetic mode). + +## 5. Windows + +No transparent unprivileged equivalent exists. AppContainer / +restricted tokens are browser-sandbox machinery — powerful, complex, +semi-documented for this use; Job objects limit resources, not file +access; Windows Sandbox is a VM feature. Anthropic's sandbox-runtime +Windows backend is alpha and needs a dedicated local user + WFP +network rules — admin setup, failing the transparency bar. **The +pragmatic path is WSL2** (already the de facto home of scientific +Python on Windows), whose Microsoft kernel ships **Landlock ABI 3** — +Windows users get the Linux enforcement path for free. Native Windows +stays out of scope (as the spec already states) and would run at +hermeticity `open`, recorded honestly. + +## 6. Syscall tracing (ReproZip et al.) — demoted to attestation + +Researched in depth before the enforcement round; retained findings: + +- **ReproZip** (NYU; releases Dec 2025 / Jan 2026, one-maintainer + bugfix cadence): ptrace tracer writing SQLite + (`opened_files` with read/write/stat/exec mode bits + canonical + paths, failed probes excluded; `executed_files` with argv/env; + full process tree). Usable without its packing feature — + WholeTale's "Recorded Runs" consumes the trace DB programmatically + in production. +- **Reliability for accidental leakage: high.** Fork/thread + auto-attach is atomic with standard ptrace options; static binaries + and mmap'd libraries are covered. Real blind spots (io_uring opens, + memfd/dlopen-from-memory, externally inherited fds) essentially + never occur accidentally in Python/numpy stacks (glibc and CPython + don't use io_uring). +- **Cost:** ~2–5× on the import storm (≈0.5–2 s), near-zero during + compute — single-digit % on real recipes. `strace --seccomp-bpf -e + trace=%file` is faster but has no structured output (you own a text + parser forever). eBPF is better and root-only — dead on HPC. +- **Role in the design:** with Landlock/Seatbelt providing prevention + on ~every venue, tracing becomes the *optional evidence layer* — + proving `traced-clean` where no boundary exists (native Windows, + exotic kernels), or auditing inside a boundary. Neither Snakemake + nor Nextflow does anything comparable (declared-IO provenance + only) — this remains a differentiator either way. +- Landlock ABI 7 (kernel 6.15) audit-of-denials is the eventual + kernel-native replacement for third-party tracing, but reading the + audit stream requires privilege — SOC-side, not `lc`-side, for now. + +## 7. Proposed design (for the next spec revision) + +**Per-output hermeticity ladder**, recorded in every manifest, never +silent: + +| Level | Meaning | +|---|---| +| `enforced` (`landlock` \| `bwrap` \| `seatbelt` \| `container`) | recipe ran inside a boundary restricted to the declared set; mechanism + effective Landlock ABI recorded | +| `traced-clean` | no boundary available; trace diff against the declared set came back empty | +| `open` | neither — `lc verify` flags it; `lc materialize --require-sandbox` refuses it | + +**Enforcement matrix:** + +| Venue | Boundary | Notes | +|---|---|---| +| Linux laptop / WSL2 | **Landlock** (default; pure-Python, ABI 1 floor, probe-and-record) → **bwrap** upgrade when usable (system-first, bundled static fallback; adds empty-world + `--unshare-net`) | the only combination that is transparent on stock Ubuntu 24.04 | +| macOS laptop | **Seatbelt** generated SBPL profile | capability-checked; fallback to trace/warn | +| Hub / GKE | the pod is the boundary; Landlock inside as defense-in-depth where the pod seccomp allows | chart adds `landlock_*` to the seccomp allowlist | +| Perlmutter | **podman-hpc** recipe wrap (site-provided); Landlock candidate pending SLE-15 probe | recipe-level wrap — no dask networking involvement | +| Native Windows | none — `open` (or WSL2 ⇒ Landlock) | out of scope | + +**Allowlist policy** (one policy, all mechanisms): project dir RW · +`.venv` + uv-managed interpreter RX · OS baseline RO (`/usr`, `/lib`, +`/etc/ssl`, locale, dyld caches on mac) · scratch/`/tmp` RW · declared +ASTRA inputs RO · **network deny by default** (a mid-recipe download +is an undeclared input), per-output opt-in via the spec. + +**Mandatory design rules** (each traces to a documented failure of a +shipped implementation): +1. **Probe, record, never silently degrade** — startup capability + probe; effective mechanism + ABI into the manifest; + `--require-sandbox` strict mode (Codex's silent-best-effort trap). +2. **Crisp denial UX** — on `EACCES`/`EXDEV`/`ENOENT`-in-sandbox, the + parent re-stats the path (stat is never blocked) and reports + "blocked by lc sandbox: `` is not part of the declared + environment", with `--no-sandbox` and `--sandbox-debug` escape + hatches (landrun's absence of this is its noted flaw). +3. **State the threat model** — enforcement of declared-dependency + *discipline* against accidental leakage; not a security boundary + against a malicious recipe (metadata visibility, + interpreter-reads-script, memfd — all named, all + adversarial-only). +4. **realpath everything** before emitting policies (macOS + `/tmp`→`/private/tmp`; symlinked venvs). + +## 8. Open verification items + +1. **Perlmutter probe**: Landlock present in SLE-15's boot LSM list? + Effective ABI? Behavior on Lustre/CFS-DVS mounts (no field reports + exist either way). One salloc session. +2. **Hub pod seccomp**: do the deployment's pods permit `landlock_*` + syscalls? (Docker default profile historically doesn't.) +3. ABI 8 / TSYNC kernel release number (man-page says 7.0 — + re-verify on release). +4. Ubuntu 26.04 LTS: does it ship the bwrap AppArmor profile by + default (25.04+ does; 24.04 does not)? +5. macOS CI smoke test for the generated SBPL profile across OS + releases. + +## Sources (primary) + +- Landlock: docs.kernel.org userspace-api/landlock + admin-guide; + landlock(7) man page (ABI table); LWN 1021648 (audit, 6.15), 1028936 + (O(1) domains); Launchpad #1950381 (Ubuntu enablement); + microsoft/WSL2-Linux-Kernel (6.6.y landlock.rst); + openai/codex codex-rs/linux-sandbox (landlock.rs, README); + landlock-lsm/rust-landlock v0.4.7; Edward-Knight/landlock; + SebastienWae/py-landlock; Zouuup/landrun; SUSE SLES-16 LSM doc. +- bubblewrap: containers/bubblewrap (releases; 0.11.2 / + CVE-2026-41163; setuid deprecation); Ubuntu userns spec SE045 + + Launchpad #2046477/#2072811; anthropic-experimental/sandbox-runtime + (+ issue #74); codex #15057/#16076; vscode #316046; melange #1508; + VHSgunzo/bubblewrap-static; moby #42441; bwrap(1). +- macOS: anthropic-experimental/sandbox-runtime (Seatbelt backend); + Codex sandbox docs (`sandbox-exec`, workspace-write); + bazel.build/docs/sandboxing; apple/containerization#737 (no removal + answer); Chromium seatbelt design doc; community SBPL references + (fG! guide, HackTricks, dnesting 2026); apple/container 1.0. +- Tracing: VIDA-NYU/reprozip (PyPI 1.3.2 Jan 2026; trace schema + docs); WholeTale Recorded Runs; arXiv 2304.08569 (strace overhead); + Gregg strace benchmarks; strace --seccomp-bpf; Neil Mitchell file- + tracing survey; Bomfather arXiv 2503.02097; Sciunit. diff --git a/docs/design/substrate-default-tradeoffs.md b/docs/design/substrate-default-tradeoffs.md new file mode 100644 index 00000000..00c34750 --- /dev/null +++ b/docs/design/substrate-default-tradeoffs.md @@ -0,0 +1,277 @@ +# Decision analysis: uv vs pixi as the default substrate — and the cost of carrying both + +- **Status:** decision analysis, v1. Weighs the substrate *posture* + question the v5.1 spec settled implicitly: uv-only, pixi-only, or + dual-substrate — and if dual, which is the default. Prompted by the + observation that the dual design's biggest cost is **maintaining two + environment types**. Draws on + [uv-vs-pixi-adoption.md](uv-vs-pixi-adoption.md), + [environment-substrate-evaluation.md](environment-substrate-evaluation.md), + the spec's pixi 0.76.2 empirical pass, this week's uv→pixi migration + experiment (pixi 0.76.2 · uv 0.12.3), and a source read of + Metaflow's environment backends (commit `2fb3c91`, 2026-08-15). +- **Date:** 2026-08-16 + +## The actual question + +"uv or pixi as default" is really three postures, because pixi's +embedded uv means pixi *alone* can cover every project, while uv +alone cannot cover polyglot ones: + +- **A. uv-only** (the v4 spec): one substrate; polyglot needs are met + by the digest-pinned BYO container escape hatch. +- **A′. uv + container hatch on demand**: one substrate; no container + at all by default; when a project needs a dependency uv cannot + source, the sandbox denial nudges it into a *declared* system layer + (`system-packages` → generated, content-addressed container run via + podman) — the Modal/Flyte ImageSpec model. See the Decision section. +- **B. pixi-only**: one substrate; pure-Python projects ride pixi's + embedded uv for the PyPI side. +- **C. dual, uv default** (the v5.1 spec): uv unless `lc init --pixi`. +- (C′. dual, pixi default — strictly dominated: it pays C's full + maintenance bill while putting the weaker-adoption tool in front of + every user. Not considered further.) + +## The dual-substrate maintenance bill, itemized + +The con deserves to be priced, not hand-waved. What "two environment +types" actually costs, per the v5.1 spec's own structure: + +| Cost | Size | Recurring? | +|---|---|---| +| Two `Substrate` implementations (9 ops each) | ~300–600 LOC + init templates | no — written once | +| Two golden fingerprint suites, two CliRunner test matrices | moderate | every engine change touching identity | +| **Empirical re-verification per tool release** | the 0.76.2 pass pinned ~10 load-bearing behaviors (`--no-install` cold-env fallback, `lock --check` writes on drift, flag-vs-env precedence, …); uv 0.12.3 has its own verified set | **yes — every supported-version bump, ×2 tools** | +| Two ambient-scrub lists (`UV_*`, `PIXI_*`/`CONDA_OVERRIDE_*`) | closed lists | audit on every tool release | +| Two hub exec-shim modes; two podman-hpc mount sets | small | venue changes ×2 | +| Detection edge cases (both manifests, both lockfiles, `[tool.pixi]`-in-pyproject) | small code, real support surface | user-facing forever | +| Per-mode asymmetries needing compensation (pixi's missing worker env-vs-lock check → mandatory env-prefix gate) | design complexity | forever | +| Docs, eval prompt, agent guidance ×2 modes | moderate | forever | +| CI needs both tools installed and warm | infra | forever | + +The one-time cost is genuinely small (Metaflow's uv backend is ~230 +lines against its conda backend's ~1,800 — backends that delegate to +a lock-owning tool are thin). The **recurring verification burden is +the real bill**: two fast-moving pre-1.0/0.x tools whose flag +semantics are load-bearing for integrity guarantees, each needing an +empirical re-pass on version bumps. That bill is capped by the +protocol design (no substrate conditional outside the 9 ops), but it +never goes to zero. + +Two honest discounts on the bill: + +- Several "pixi compensations" are good hygiene uv-mode already wants + (env-prefix existence check, explicit project pins, ambient scrub) — + they'd survive a pixi deletion. +- The fabric, manifest chain, sandbox, and CLI surface are + substrate-blind by construction; the 2× is confined to the edge. + +## A. uv-only + +**Pros** + +- **One environment type** — the entire bill above halves; one tool + to empirically track, one scrub list, one test matrix. +- The default path keeps every adoption advantage: agents know uv + from training data (the eval failures were agent + environment-boundary confusion — "the agent already knows the tool" + is a reliability property, not a popularity contest); ~196M + downloads/month; native IDE/Dependabot/CI support; zero extra + install; `lc` itself arrives via `uv tool install`. +- Battle-tested prior art for exactly this architecture + (Flyte/Union, Metaflow, Modal, ClearML, Ray's uv hook). +- Current workloads don't exercise the conda layer: dask-over-TCP (no + MPI), PyPI CUDA wheels, vendored BLAS — structural for MPI, + verified for the fabric. + +**Cons** + +- Polyglot projects (R, Julia, TeX, compilers, HDF5) get a materially + worse story: system tools become README prose + host attestation, + or a BYO container — reintroducing exactly the container-authoring + burden the design deleted, for the projects least equipped to + carry it. +- The sandbox's sharpest guarantee (G6: undeclared tool ⇒ loud + failure) loses its remedy in uv mode: lc can *catch* the host + `latex` leak but can't offer a declared home for `latex`. Enforcement + without a fix path invites `--no-sandbox` habituation. +- Interpreter identity stays attestation-grade (version pinned, build + as residue) rather than content-addressed. +- The unsurveyed presumption: "no current project needs conda-forge" + has not been checked against actual projects. If wrong, uv-only + fails the very users the sandbox was built to protect. + +## B. pixi-only + +**Pros** + +- **One environment type** — same halving of the bill, and no + detection logic, no migration story, no one-way door, no + mode-asymmetry table in the docs. +- Strictly stronger identity everywhere: interpreter build + content-addressed in the lock; system layer (conda-forge, URL + + sha256) in the lock; channels and indexes inside the lock bytes. +- The sandbox story is uniformly clean: every denial has the same + one-line fix (`pixi add `); the exec-tier allowlist debate + shrinks. +- PyPI resolution quality is uv's own (pixi embeds uv) — verified to + resolve byte-identical PyPI artifacts in the migration experiment. +- Self-contained envs (`env -i /bin/python` verified) simplify + the hub shim and the podman-hpc mount set to one mode. + +**Cons** + +- **Every project pays pixi's costs, including the pure-Python + majority**: the extra binary bootstrap on every venue (not + pip-installable), conda-forge as a mandatory channel (bigger envs, + channel churn in the lock), and pixi's verified traps + (`--no-install` silent host-PATH fallback on cold envs, + `lock --check` writing on drift) on the *default* path rather than + the opt-in path. +- Adoption is an order of magnitude behind and not closing: absent + from usage surveys, ~12 Stack Overflow questions, a 5-person + pre-1.0 vendor. For an agent-first tool this is the decisive + reliability gap — agents reach for `uv add`/`uv run` unprompted and + must be *retrained by prompt* into pixi verbs on every project. +- No battle-tested workflow-system precedent: Metaflow, Flyte, Modal, + ClearML, Ray — all uv, none pixi (verified against Metaflow source; + no pixi backend exists anywhere in that tier). lightcone would be + first, alone, on the vendor's schedule. +- Sustainability risk concentrates in one small vendor; uv-only risk + concentrates in Astral, which is VC-funded but an order of magnitude + more entrenched. +- Weaker worker-side verify (no true env-vs-lock no-write check) + becomes the *only* posture, not the exceptional one. + +## C. Dual, uv default (v5.1) + +**Pros** + +- Each project gets the right tool: the majority path keeps uv's + adoption/agent advantages at zero extra cost; polyglot projects get + first-class locked system layers instead of containers or prose. +- The sandbox guarantee keeps a remedy in both modes (declare in + pixi; convert or containerize in uv). +- Preserves optionality: if pixi's ecosystem position improves (or + collapses), the default can move (or the mode can be deleted) + without an architecture fork — the protocol is the hedge. + +**Cons** + +- **The maintenance bill above** — dominated by the recurring + two-tool empirical verification, forever. +- A user-facing mode split: two init paths, two dependency verbs, an + asymmetry table users eventually meet ("why is worker verify weaker + here?"), detection edge cases, and a one-way migration door. +- Intersection semantics: lc's guarantees cover only the default env + in both modes — the contract must be stated twice and enforced at + the sandbox layer. +- Risk of the untested half: if few projects actually choose pixi + mode, it becomes the rarely-exercised branch — precisely where + integrity bugs hide. (Mitigation: CI parity, pixi-mode eval + coverage — which is itself more of the bill.) + +## Where the evidence points + +The decision hinges on one unmeasured quantity: **what fraction of +real Lightcone projects need the conda-forge layer.** The rationale +doc already flags this survey as open; it is the deciding datum: + +- **≈0%** → A (uv-only) wins: the bill halves, the BYO-container + escape hatch covers the tail, and nothing of value is lost. The + cost of being wrong later is bounded — the substrate protocol can be + re-added (it was designed once; the pixi empirical pass is + documented). +- **A small-but-real minority (the presumed case)** → C stands, and + the bill is the price of first-class polyglot without making the + majority pay pixi's costs. Keep the bill capped: support exactly one + pinned version range per tool, gate bumps on the empirical + checklist, and require pixi-mode CI/eval parity from day one. +- **A majority** → B becomes thinkable — but only if the agent + problem is solved by prompting (the eval must demonstrate agents + operating pixi verbs as reliably as uv verbs), since the eval + evidence is the design's origin. Adoption trend would need to have + turned as well. + +A useful asymmetry when weighing A vs C: **deleting pixi mode later +is cheap; adding it later is also cheap** (the protocol and the +empirical pass are the expensive artifacts, and both now exist on +paper). The genuinely expensive commitment would be B — betting the +default on the smaller ecosystem — because walking *back* from +pixi-default means migrating every project across a one-way door in +reverse (conda deps have no uv home). + +## Recommendation (superseded by the Decision below) + +Run the project survey before paying another increment of the dual +bill. Until it lands: keep C (dual, uv default) as specified, but +treat pixi mode as **frozen at one pinned version** (no +version-range chasing) and make the survey a migration-step-7 +prerequisite — if the survey comes back empty, ship stage 2 as A +(uv-only) and leave the pixi implementation as a documented, +evidence-backed re-add rather than shipped code. + +## Decision (2026-08-16): A′ — uv + container hatch on demand + +Adopted as **spec v6** +([execution-environment.md](execution-environment.md)). A′ refines A +with an escalation path that keeps G6's denial actionable: the +default is pure uv with zero extra installs; declaring +the `[tool.lightcone.image]` table (or `Containerfile.extra`) flips +the project into containerized mode — lc renders the lock + declared +system layer into a content-addressed image (never a user-authored +root Containerfile), run via podman / podman-hpc / pods. + +Why A′ over C (dual substrate), beyond the bill above: + +- **Every posture asks the user to install exactly one extra tool** + — pixi in C, podman in A′. Podman is the install that generalizes: + a project that grows complex and deploys to HPC or k8s wants a + container eventually anyway, so the escalation converges with where + such projects were headed; pixi is a dead-end install by + comparison. (And a container runtime is arguably baseline developer + tooling in a way pixi is not.) +- **Shipped precedent instead of none**: uv-lockfile-first + derived + container with a declared apt layer is exactly Modal + (`Image.apt_install(...).uv_sync()`), Flyte/Union ImageSpec, and + the pattern Metaflow ships — no workflow system ships pixi. +- **Reversible escalation**: deleting the declaration returns the + project to direct mode; the uv→pixi migration was a verified + one-way door. +- **Agent familiarity on both rungs**: agents know uv *and* + apt/containers from training data; pixi verbs they do not. +- Honest costs, accepted: apt is name-pinned only (dpkg-snapshot + attestation, snapshot-pinning as future hardening) — weaker + system-layer identity than `pixi.lock`'s content-addressing; macOS + escalation means a `podman machine` VM (linux builds, no GPU); G4 + requires probes to run in-container once escalated. + +Standing pixi re-add triggers (unchanged in substance): the project +survey reveals a meaningful laptop-centric conda-layer population +for whom a container VM is unacceptable; or pixi reaches 1.0 with an +adoption inflection. The pixi 0.76.2 empirical pass is retained in +the spec's evidence appendix so a re-add starts from documented +ground, not from scratch. + +## Amendment (2026-08-17): full-stack containerized mode (spec v6.1) + +The v6.0 form of A′ wrapped only the *recipe* in the container while +the engine and workers synced the full lock on the bare host. The +multi-agent review +([execution-environment-v6-review.md](execution-environment-v6-review.md)) +confirmed this deadlocks on lock-level system dependencies (rpy2 +needing R, sdist builds needing headers) — the hatch's own examples. +v6.1 adopts the scope-preserving resolution: **in containerized mode +the entire execution stack (driver, workers, recipes, probes) runs +inside the image**; no host environment exists for the project. This +closes the deadlock (uv sync runs inside the image build, after the +apt layer), preserves engine-version coherence, deletes the dual +host/image environment, and — because the container is Linux on +every host OS — enables per-recipe Landlock scoping even on macOS. +Costs, accepted: Perlmutter containerized requires the podman-hpc +full-stack worker launch (`--net=host`, GPU-via-CDI spike-gated), +and the hub deployment contract gains a +notebook-image-`FROM`-project-image item. This strengthens the +"podman generalizes" argument above: the same install now carries a +project from laptop escalation to HPC and k8s without an +architecture change. diff --git a/docs/design/uv-vs-pixi-adoption.md b/docs/design/uv-vs-pixi-adoption.md new file mode 100644 index 00000000..5e995c01 --- /dev/null +++ b/docs/design/uv-vs-pixi-adoption.md @@ -0,0 +1,286 @@ +# Findings: uv vs pixi adoption, and lockfile-first patterns in other workflow systems + +- **Status:** findings report — supplementary evidence for revising + [execution-environment.md](execution-environment.md) toward a + uv-based substrate +- **Date:** 2026-08-15 +- **Method:** 4 parallel web-research agents (uv adoption metrics, pixi + adoption metrics, environment handling across 15+ workflow/ML-infra + systems, uv's scientific-computing gaps and the wheel-variants + standards track). Primary sources preferred; unverified claims + flagged inline. + +## TL;DR + +The adoption gap is roughly **an order of magnitude on every measurable +axis**, and it is widening. uv is the presumptive default for new +Python projects (32% of Python repos created in 2025 ship a `uv.lock`); +pixi does not register in any usage survey. The "lockfile as source of +truth → container derived mechanically, tagged by a hash of the lock" +pattern the current design adopts via pixi **already exists in +production form built on uv**: Flyte/Union's ImageSpec, Metaflow +`--environment=uv`, Modal `Image.uv_sync`, ClearML agent — all shipped +in 2025. The scientific workflow managers (Snakemake, Nextflow) support +neither tool and remain conda-first. + +The technical gap the substrate evaluation identified — uv.lock pins +only the Python layer — is unchanged in kind but has narrowed +materially on the CUDA axis and is precisely the gap a derived +container closes. A **uv + derived-container design** (the evaluation's +own named degradation path, candidate D with uv substituted) is +supported by stronger prior art than the pixi variant, at the cost of a +composite environment identity (lock hash + image identity) instead of +one lockfile that pins everything. + +## 1. Adoption: the numbers + +| Axis | uv (Astral) | pixi (prefix.dev) | +|---|---|---| +| GitHub stars | **88,746** | 7,567 | +| Contributors | 468 | 284 | +| Downloads | **~196M/month** (PyPI alone, pypistats.org 2026-08); most installs arrive via other channels and are not even counted | **~17.3M cumulative ever** (GitHub releases, all 142 releases summed); ~100k+/release-week currently | +| Survey presence | 11% in JetBrains/PSF 2024 survey (from 0% the year before, >30k respondents); most-admired technology in Stack Overflow 2025 (74.2%) | **absent from every survey found** — JetBrains, SO, conda-ecosystem; no survey reports a pixi share at all | +| New-project penetration | `uv.lock` in **32% of Python repos created in 2025**, 30% of 2026-Q1 repos (aleyan.com census of top-100k GitHub repos) | ~8.5k `pixi.lock` files on all of GitHub (code search); ~5k workflows use `setup-pixi` | +| Bot/ecosystem support | Dependabot **native**, Renovate native, PyCharm 2025.3 **default backend**, GCP Buildpacks default installer (Python ≥3.14), Databricks bundles require it, Airflow docs list "pip or uv" | Renovate native; **Dependabot: no** (open since 2023); no native PyCharm/VS Code support (conda-shim workaround) | +| Stack Overflow | (not counted — large) | **12 questions** total on the tag | +| Version / stability | 0.12.5 (2026-08-14); still no 1.0 | 0.76.2 (2026-08-10); still no 1.0, **no public 1.0 roadmap found**; lock format v7 (bumped May 2026) | +| Company | Astral — **acquired by OpenAI, March 2026**; commitment to keep uv/Ruff/ty open source; pyx registry wound down and its GPU-packaging infra open-sourced (June 2026) | prefix.dev — **5 people**, Berlin; undisclosed seed (2022, 468 Capital + Costanoa); Pro plan €42/mo; rattler library transferred to the conda org (Oct 2024) | + +Where pixi *is* adopted, it is deep and domain-specific: conda-forge's +own maintainer tooling (`pixi run rerender`, `conda_install_tool: +pixi`), robotics (RoboStack "recommended for new installations", Open +Robotics standardized on it for Windows dev), QuantCo (40+ repos in +production), and the SciPy 2025 tutorial track (two pixi tutorials, one +now a SciPy Proceedings paper). An arXiv paper (2511.04827) claims +~5,300 adopting projects. + +Corrections to the substrate evaluation's context: + +- **NERSC does not mention pixi (or uv) anywhere in its Python docs** — + the evaluation already flagged the pixi-at-NERSC story as + operationally unverified; this confirms there is no site endorsement + on either side. NERSC still teaches conda/mamba + containers at + scale. +- Among centers that do name a tool: **CSCS explicitly documents uv** + ("pip, uv and Conda can all be used", uv venv examples, squash-into- + image guidance), ORNL's ExCL docs recommend uv, UF HiPerGator has a + full uv endorsement page (with "use pixi for non-Python deps" as the + caveat). No HPC center endorses pixi by name. +- The scientific-Python teaching consensus (pyOpenSci guide, Scientific + Python dev guide) is **uv as the primary recommendation, pixi when + conda-forge binaries are needed** — and pixi *embeds uv* as its PyPI + resolver, so the community frames them as complements, not rivals. + +### Sustainability reading + +Both tools are pre-1.0. The risks differ in kind: pixi's is +concentration (5-person VC-funded company, no 1.0 roadmap, though the +lock format is open and rattler now lives in the conda org); uv's is +governance (OpenAI now owns Astral — resources are no longer a +question, but priorities could shift; the stated open-source commitment +and the open-sourcing of pyx's GPU packaging are the positive signals, +and uv's adoption is now so broad that a community fork would be viable +if needed). On the "will an agent or a new user already have and know +this tool" axis — the axis that matters for lightcone's agent-facing +UX — uv wins outright: it is in every harness's training data, ships in +agent sandboxes, and is how `lc` itself is installed today. + +## 2. Prior art: who has shipped what (2025–2026) + +The substrate evaluation surveyed prior art through a conda-centric +lens (Nextflow Wave, nf-core, Snakemake pin-files). A second tier of +systems has since converged on the *same lockfile-first shape built on +uv*: + +| System | uv integration (shipped) | Pattern | +|---|---|---| +| **Flyte / Union.ai** | `ImageSpec(requirements="uv.lock")`; Flyte v2 `Image.with_uv_project(pyproject_file=…, uvlock=…)` | **Container derived from the lock; image tag = deterministic hash of spec + lock content + source root; registry checked before rebuild.** The closest existing analogue to lightcone's content-addressed `lc--` scheme — with the identity flaw already fixed. | +| **Metaflow** (2.15.8, May 2025) | `--environment=uv` packages the whole uv project (pyproject + uv.lock) and re-materializes it on Kubernetes/Batch workers | Lock executed frozen on remote workers. Caveat: forgoes Metaflow's content-addressed S3 env snapshots (their availability guarantee), per-flow not per-step. | +| **Modal** (July 2025) | `Image.uv_sync()` — runs `uv sync --frozen` from pyproject + uv.lock in a server-side derived image | No user Dockerfile; images content-addressed from the spec. | +| **ClearML agent** (v1.9/3.0) | `package_manager.type: uv`; if the repo has a uv.lock, the agent runs `uv sync --locked`, overriding the pip-freeze snapshot | Lock as authoritative env on remote execution. | +| **Dagster** | `uvx create-dagster` canonical; "uv is the future of Python project management for Dagster projects" (maintainer statement) | Scaffolding-level only; isolation unit is the code-location venv. | +| **Coiled** | package sync scans the local (uv-managed) env and replicates deterministically on cluster VMs | Transparent replication; lockfile recommended client-side. | + +Meanwhile the scientific WMS tier has shipped **nothing** for either +tool: Snakemake's uv issue (#3251) is open-stale with no maintainer +response; its pixi issue (#3915) is blocked on the software-deployment +plugin framework (PR #3339), still an open draft at ~17 months; +Nextflow has no uv issue at all and its direct-pip provider requests +(#4664, #4671) remain unshipped. This doesn't hurt lightcone — the +design already keeps Snakemake env-oblivious — but it removes any +"align with Snakemake's pixi direction" argument: that direction is +indefinitely stalled either way. + +Supporting infrastructure for uv.lock-as-identity is first-class and +officially documented: + +- `hashFiles('uv.lock')` as cache key is Astral's own documented CI + pattern; `setup-uv` computes cache validity from the lock hash. +- The multi-stage Docker pattern (SHA-pinned `ghcr.io/astral-sh/uv` + binary, copy pyproject + uv.lock, `uv sync --frozen + --no-install-project` deps layer, copy source, final stage copies + only `.venv`, `UV_PYTHON_DOWNLOADS=never`) is the de-facto standard + Python container build, replicated across Astral docs, Hynek + Schlawack, Microsoft ISE, Depot. +- `uv2nix` reproduces a uv.lock bit-for-bit as Nix derivations. +- `uv export --format requirements.txt` (with hashes) bridges to + pip-only contexts. + +No dedicated "Dockerfile generator from uv.lock" tool exists — lightcone +generating one is exactly the thin residual code every surveyed system +also keeps. + +## 3. The technical gap, re-measured + +The evaluation's core objection to uv-canonical stands in kind: uv.lock +pins wheels and the interpreter, not BLAS/MPI/HDF5/compilers. What has +changed: + +- **CUDA (much better).** NVIDIA now publishes an official + `cuda-toolkit` metapackage on PyPI (13.3.1, June 2026) with extras + including **nvcc itself**, cudart, cublas, cusolver, cufft — the + toolkit is pip/uv-installable and lockable. uv's `--torch-backend` + (`auto`/`cu118`…`cu130`/rocm/xpu) plus `[tool.uv.index]` with + platform markers is the documented PyTorch path. The host *driver* + remains unpinned (true under pixi and containers too). Known footgun: + `--torch-backend=auto` on a GPU-less build host silently selects CPU + wheels — pin explicitly in any container build. +- **Wheel variants (the 2027 fix, not the 2026 one).** PEP 817 → PEP + 825 (package format, split Feb 2026) are active Drafts with authors + from NVIDIA, Meta/PyTorch, Astral, Quansight, Anaconda; a major PEP + 825 revision landed 2026-08-13 and the delegate (Paul Moore) is + reviewing favorably. Experimental variant-enabled uv and PyTorch 2.8 + variant wheels exist. Mainline uv has **not** shipped variant + support; realistic ecosystem readiness is 2027. Variant-based + MPI pinning has no provider at all yet. +- **MPI (unchanged).** mpi4py against Cray MPICH is still an sdist + build with `MPI4PY_BUILD_MPICC`/`--no-binary` against the host `cc` — + uv passes these through (`[tool.uv] no-binary-package = ["mpi4py"]`), + but the toolchain step is unlocked, and no HPC center documents the + uv variant of the incantation yet. On the container path this is + moot: NERSC's podman-hpc + `cray-mpich-abi` is the site-blessed + mechanism regardless of what installed the Python layer. +- **BLAS variant selection (unchanged).** Impossible on PyPI — numpy/ + scipy wheels vendor one OpenBLAS. If a project needs MKL vs OpenBLAS + control outside a container, only conda-forge provides it. + +## 4. Implications for the design decision + +What the new evidence changes in the evaluation's weighing: + +1. **The prior-art claim inverts.** The evaluation found the + lockfile-first pattern "converged" in conda-flavored systems. The + 2025 wave shows the *same* pattern shipped on uv by the ML-infra + tier — and Flyte's ImageSpec is a closer match to lightcone's + content-addressed image scheme than anything in the conda camp. + uv-lockfile-first + derived container is established practice, not a + novel construction. +2. **Agent/user familiarity was underweighted.** The eval scored + candidates on install bar but not on "does the agent already know + this tool". Every coding agent has deep uv training data and most + sandboxes ship it; pixi is a long-tail tool an agent must be taught. + For a product whose primary UX is agent-driven, this is a real + scoring axis, and it is lopsided. +3. **The vendor-risk comparison shifted.** Astral's acquisition + resolves its sustainability question (while raising a governance + one); prefix.dev remains a 5-person pre-1.0 company with no public + 1.0 commitment. The eval's mitigation for pixi risk was "degrade to + the uv variant of D" — the new evidence suggests starting there. +4. **The identity cost is real and must be stated honestly.** One + pixi.lock pins interpreter + BLAS + MPI + system libs in a single + per-platform artifact; `env_version` = one lock projection hash. + Under uv + container, the environment identity is **composite**: + `env_version = H(uv.lock projection)` for the Python layer, plus — + whenever the container transport is used or system deps are declared + — the derived image identity (digest-pinned base + generated + Containerfile + any system-package declaration). Sites that need + the system layer pinned (GKE, Perlmutter at scale) are exactly the + sites where the container transport runs, so the pinning lands where + it is needed; the residue is direct-transport execution relying on + host system libs (host BLAS is *not* in play — PyPI wheels vendor + their own — but host glibc, MPI, and any non-Python tools are). + The manifest's existing `transport` + image-digest fields carry + this honestly. +5. **What is genuinely lost vs pixi:** single-file whole-stack pinning; + conda-forge's `mpich=*=external_*` no-container MPI story at small + scale (under uv, vendor-MPI mpi4py on the direct path is an unlocked + host build — record as attestation, or push MPI workloads to the + container transport); BLAS-variant control outside containers; and + non-Python tools (TeX etc.) always require the container escape + hatch rather than a conda dependency. None of these affect the + laptop/pure-Python majority path; all of them have a container + answer. + +### Sketch: candidate D-uv (uv + derived container) + +- `pyproject.toml` + `uv.lock` (+ `.python-version` pinning an exact + patch) are the committed source of truth. `tool.uv.required-environments` + forces lock-time failure for platforms lacking wheels. +- `lc run CMD` wraps `uv run --frozen`. Stale lock → pointed error, + never silent re-resolve. Same boundary rule, litmus test reworded to + "fix `pyproject.toml`". +- `env_version` = hash of a normalized uv.lock projection (sorted + artifact URLs + sha256s). uv.lock is universal (one file, all + platforms), so unlike pixi the hash does not vary per platform — + still record `platform` in the manifest since resolved binaries + differ. +- System-level needs are declared, not authored: a small optional + declaration (e.g. apt packages / CUDA toolkit extras) feeds the + generated multi-stage Containerfile (digest-pinned base, Astral's + documented uv pattern). Declaring any system dep — or a site + requiring image transport — routes materialization through the + container, whose identity joins the manifest. `Containerfile.extra` + escape hatch as in the current draft. +- Transports unchanged from the draft: direct (laptop, small HPC) / + podman-hpc (Perlmutter at scale) / kubernetes (GKE). Pre- + materialization warm-install rule carries over (`uv sync --frozen` + driver-side; uv's cache is designed for concurrency but the + warm-before-fan-out rule stays cheap insurance on Lustre). +- Bootstrap story collapses: uv is the one tool, it installs `lc` + itself (`uv tool install lightcone-cli`), no binary bootstrap + machinery, no pixi-version pin in config. + +### Open items this evidence does not settle + +- Whether any current lightcone project actually needs BLAS-variant + control or no-container vendor MPI at small scale — the two cases + where pixi is strictly stronger. If yes, the pixi design stands on + its merits; if no, D-uv covers the real workload with the more + adopted tool. (The NERSC spike in the design's open questions would + answer the MPI half — now for the container path instead.) +- GPU lock ergonomics from mac laptops (torch-backend pinning replaces + `CONDA_OVERRIDE_CUDA`; needs a concrete `lc init` default). +- Re-check wheel variants (PEP 825) around early 2027 — if accepted and + shipped in uv, the CUDA/MPI pinning story changes again in uv's + favor. + +## Sources (primary, fetched 2026-08-15) + +- uv metrics: pypistats.org/packages/uv; api.github.com/repos/astral-sh/uv; + lp.jetbrains.com/python-developers-survey-2024; survey.stackoverflow.co/2025; + aleyan.com/blog/2026-why-arent-we-uv-yet (repo census); + openai.com/index/openai-to-acquire-astral + CNBC/Bloomberg 2026-03-19; + pydevtools.com/blog/astral-winds-down-pyx-open-sources-gpu-packaging +- pixi metrics: api.github.com/repos/prefix-dev/pixi (7,567 stars, 142 + releases, 17.3M asset downloads summed); api.anaconda.org/package/conda-forge/pixi; + formulae.brew.sh/api/formula/pixi.json; GitHub code search + (setup-pixi 5,048 workflows; 8,512 pixi.lock files); StackExchange + API (12 questions); tech.quantco.com/blog/pixi-production; + robostack.github.io; arXiv 2511.04827; conda.org/blog/2024-10-01-rattler-to-conda +- Workflow systems: docs.metaflow.org/scaling/dependencies/uv; + docs.flyte.org ImageSpec + union.ai/docs/v2 (with_uv_project, tag + hashing); modal.com/docs/guide/images (uv_sync); + clear.ml/docs clearml_agent_execution_env; + github.com/snakemake/snakemake #3251 #3915 PR#3339; + github.com/nextflow-io/nextflow #4664 #4671 #5219; + docs.astral.sh/uv/guides/integration/docker + github + dependabot + renovate; + github.com/pyproject-nix/uv2nix +- Scientific gaps: pypi.org/project/cuda-toolkit (13.3.1, nvcc extra); + docs.astral.sh/uv/guides/integration/pytorch (torch-backend); + peps.python.org/pep-0817 + pep-0825 (Drafts; 825 revised 2026-08-13); + astral.sh/blog/wheel-variants; pytorch.org/blog/pytorch-wheel-variants; + mpi4py.readthedocs.io/en/stable/install; docs.nersc.gov (no uv/pixi); + docs.cscs.ch/build-install/python (uv documented); + docs.rc.ufl.edu/software/uv; docs.excl.ornl.gov quick-start; + pyopensci.org python-package-guide; learn.scientific-python.org + dev-environment; pydevtools.com uv-vs-pixi-vs-conda diff --git a/docs/hpc/containers.md b/docs/hpc/containers.md deleted file mode 100644 index 786dc86d..00000000 --- a/docs/hpc/containers.md +++ /dev/null @@ -1,52 +0,0 @@ -# Container Builds for HPC - -HPC nodes generally cannot reach a docker daemon, so lightcone-cli ships -support for `podman-hpc` (NERSC Perlmutter and friends). The build/migrate -workflow is owned by `lightcone.engine.container`. - -## podman-hpc workflow - -`podman-hpc` is rootless and HPC-aware. After a build, the image must be -*migrated* into the per-node container cache so compute nodes can read it -without a registry. - -```bash -# On a login node, with podman-hpc on PATH: -lc setup # writes ~/.lightcone/config.yaml -$EDITOR ~/.lightcone/config.yaml # set container.runtime: podman-hpc -lc build # builds + migrates each image -``` - -`lc build` checks for cached tags and skips rebuilds. Use `--force` to -rebuild everything. - -## Tag computation - -Tags are content-addressed: - -``` -lc-- -``` - -The hash covers the Containerfile contents plus any of these dependency -files found at the project root: -`requirements.txt`, `requirements-dev.txt`, `requirements-test.txt`, -`pyproject.toml`, `setup.py`, `setup.cfg`, `poetry.lock`, `Pipfile.lock`. - -## At run time - -`lc run` does **not** re-shell into Snakemake's `container:` directive or -`--sdm apptainer`. The Snakefile generator wraps each rule's recipe in: - -```bash -podman-hpc run --rm --pull=never -v "$PWD":"$PWD" -w "$PWD" \ - bash -c '' -``` - -`--pull=never` is critical: short-name resolution would otherwise try -`unqualified-search-registries` for tags like `lc-myproject-abc123` and -fail. Pre-pulling registry images via `lc build` (or pre-staging -Containerfile images via `lc build`) is therefore mandatory. - -See also: [api/container](../api/container.md) for the implementation, -and [`lc build`](../cli/build.md) for the user-facing command. diff --git a/docs/hpc/index.md b/docs/hpc/index.md deleted file mode 100644 index 672e3fc1..00000000 --- a/docs/hpc/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# HPC & SLURM (consolidated) - -The standalone HPC subsystem (target files, site registry, sbatch -generation) is gone. SLURM execution is now handled by Dask: when `lc run` -is invoked inside an existing SLURM allocation, the cluster manager -launches one `dask worker` per allocated node via `srun` and Snakemake -dispatches each rule across them. - -For the user-facing flow, see [Running on a Cluster](../user/cluster.md). - -For maintainer detail: - -- [api/dask_cluster](../api/dask_cluster.md) — the three-branch decision - (existing scheduler / SLURM allocation / local). -- [api/dask_executor](../api/dask_executor.md) — the Snakemake executor - plugin that turns each rule into a `client.submit(...)` call. -- [api/container](../api/container.md) — `podman-hpc` build & migrate. diff --git a/docs/hpc/site-registry.md b/docs/hpc/site-registry.md deleted file mode 100644 index 49dd39e7..00000000 --- a/docs/hpc/site-registry.md +++ /dev/null @@ -1,6 +0,0 @@ -# Site Registry (orphaned) - -The `lightcone.engine.site_registry` module still exists but is not imported -by any active code path. It carries Perlmutter scheduler defaults that used -to feed the wizard for the (now removed) target system. See -[api/site_registry](../api/site_registry.md) for the current state. diff --git a/docs/hpc/targets.md b/docs/hpc/targets.md deleted file mode 100644 index 2db4548c..00000000 --- a/docs/hpc/targets.md +++ /dev/null @@ -1,3 +0,0 @@ -# Target Configuration (removed) - -The per-machine target system is gone. See [`lc target`](../cli/target.md). diff --git a/docs/maintainer.md b/docs/maintainer.md index 0cbdd3c6..326ab299 100644 --- a/docs/maintainer.md +++ b/docs/maintainer.md @@ -9,14 +9,12 @@ If you're looking for the user-facing docs, the ## What this covers -- [Architecture](architecture.md) — the three subsystems (Snakefile generation, - manifest layer, cluster management) and the invariants that hold them together. -- [CLI Reference](cli/index.md) — every `lc` command: flags, options, and the - exact Snakemake invocation each one triggers. -- [Python API](api/index.md) — the `lightcone.engine.*` modules: public - signatures, common entry points, and module responsibilities. -- [HPC & SLURM](hpc/index.md) — how the Dask cluster manager adapts to local, - SLURM, and external schedulers. +- [Architecture](architecture.md) — the environment, integrity, and + hermeticity layers, the image hatch, and the invariants that hold + them together. +- [CLI Reference](cli/index.md) — every `lc` command: flags, options, + and what each one actually does. +- [Python API](api/index.md) — the `lightcone.*` module map. - [Contributing](contributing/setup.md) — clone, install, run the test suite, lint, and build the docs locally. diff --git a/docs/user/cluster.md b/docs/user/cluster.md deleted file mode 100644 index ca06fb85..00000000 --- a/docs/user/cluster.md +++ /dev/null @@ -1,270 +0,0 @@ -# Running on a Cluster - -When local laptop time isn't enough, you can take the same project to -a SLURM HPC system or a lightcone JupyterHub deployment. There's no -separate configuration to learn — the same `lc run` command works -everywhere, just with more hardware to spread across. - -## The big picture - -`lc run` always dispatches through a Dask cluster. Four branches: - -1. On your laptop → a `LocalCluster` sized to the machine. -2. **On a JupyterHub deployment** (Dask Gateway detected) → a - run-scoped Gateway cluster created with your project's container - image and shut down when the run finishes. -3. **Inside a SLURM allocation** → an in-process scheduler bound to - the driver's hostname, with one `dask worker` per allocated node - launched via `srun`. -4. With `DASK_SCHEDULER_ADDRESS` set → connect to whatever scheduler - you've pointed at. - -You don't pick — `lc run` detects which case applies. The only thing -you do differently on a cluster is request the nodes (and on -JupyterHub, not even that). - -## JupyterHub deployments (Kubernetes + Dask Gateway) - -On a lightcone JupyterHub (GKE with Dask Gateway and Cloud Build), -everything is zero-configuration — the deployment injects the whole -contract into your session (`DASK_GATEWAY__*`, `LIGHTCONE_REGISTRY`, -`LIGHTCONE_BUILD_BUCKET`), and `lc` picks it up: - -- The scaffolded project image doubles as the Dask worker pod image - with no hub-specific content: `lc init` pins `lightcone-cli` in - `requirements.txt`, which brings the whole execution stack - (snakemake, dask, distributed, dask-gateway) on top of your own - dependencies — the same image runs anywhere. -- `lc build` builds through the deployment's **GCP Cloud Build** - service (there's no docker in your session) and pushes - `/lc-:` to the hub's Artifact - Registry. Unchanged files never rebuild — freshness is one registry - check. -- `lc run` makes sure the image is up to date, **creates a Dask - Gateway cluster with that image**, runs the pipeline in worker pods - (recipes execute directly in the image — no nested containers), and - **culls the cluster when the run finishes**. Your NFS home is - mounted in every worker pod at the same path, so outputs land in - the project tree exactly as they do locally. - -A cluster's image is fixed at creation, so create-per-run is also what -keeps the environment fresh: edit the Containerfile, `lc run`, and the -next cluster runs the rebuilt image. - -## Pre-flight: pick the right container runtime - -On most HPC sites, docker isn't available on compute nodes. Most -SLURM systems (including NERSC Perlmutter) provide `podman-hpc`. On a -login node: - -```bash -$EDITOR ~/.lightcone/config.yaml -``` - -```yaml -container: - runtime: podman-hpc -``` - -Then build and migrate the images for your project: - -```bash -cd my-analysis -lc build -``` - -`lc build` runs `podman-hpc build` and then `podman-hpc migrate`, -which copies the image into the per-node container cache. Compute -nodes can read it without registry access. - -If your site has only `apptainer` / `singularity`, the Lightcone -toolchain doesn't ship explicit support for those today — you can run -without containers (`runtime: none`) for the moment, with the caveat -that the manifest's `container_image` field will record what was -declared, not what executed. (See [`lc run`](../cli/run.md) for the -provenance warning.) - -## A typical SLURM workflow - -### 1. Get an allocation - -```bash -salloc -N 4 -t 02:00:00 -C gpu # interactive -# or -sbatch run.sbatch # batch -``` - -`run.sbatch` looks like: - -=== "Generic" - ```bash - #!/bin/bash - #SBATCH -N 4 - #SBATCH -t 02:00:00 - #SBATCH -C gpu - - cd $HOME/my-analysis - source .venv/bin/activate - lc run -j 16 - ``` - -=== "NERSC Perlmutter" - ```bash - #!/bin/bash - #SBATCH -A - #SBATCH -q regular - #SBATCH -C gpu - #SBATCH -N 4 - #SBATCH -t 04:00:00 - - cd $SCRATCH/your-analysis - - # make `lc` available — pick the line that matches your install: - export PATH=$HOME/.local/bin:$PATH # uv tool install - # source ~/.conda/envs/your-env-name/bin/activate # conda env - - lc run -j 16 - ``` - -### 2. `lc run` inside the allocation - -Once `SLURM_JOB_ID` is set in your environment, `lc run` does the rest: - -- Starts an in-process Dask scheduler bound to the SLURM node hostname. -- Launches one `dask worker` per node via `srun`. -- Each worker advertises the node's CPU, memory, and GPU resources. -- Snakemake submits each rule via the Dask executor; rules with - per-recipe `resources:` constraints land on workers that can hold - them. - -### 3. Per-recipe resource hints - -Add resource hints in your `astra.yaml` recipe blocks: - -```yaml -outputs: - - id: heavy_fit - type: metric - recipe: - command: python scripts/fit.py --output {output[0]} - resources: - cpus_per_task: 32 - mem_mb: 64000 - gpus_per_task: 1 -``` - -The Snakemake-via-Dask executor maps these to per-task resource -requests, so a rule that needs a GPU only schedules on nodes that -advertise one. - -## Interactive: iterating inside an allocation - -During development you're usually iterating — run something, check the -result, adjust the spec, repeat. For that loop you want an interactive -shell inside a SLURM allocation, so that `lc run` executes on the -compute node rather than the login node. - -```bash -salloc -A -q interactive -C gpu --nodes=1 -t 00:30:00 -# salloc drops you onto a compute node; from there: -cd /path/to/your-analysis -lc run --universe baseline -lc status -``` - -Everything you launch from that shell (`lc run`, scripts, etc.) -executes on the allocated node. When you're done iterating and want a -hands-off sweep of all universes, submit `lc run` as a batch job -instead (the sbatch template above). - -## What about login-node-only operations? - -Build images, dry-run, look at status — all fine on a login node -without an allocation: - -```bash -lc build # build images (uses podman-hpc on login node) -lc status # offline; reads only manifests -``` - -The actual `lc run` should happen inside an allocation, since that's -where the worker nodes are. - -## External Dask schedulers - -If you have a long-lived Dask cluster (Slurm jobqueue, k8s, etc.) -that you'd rather attach to: - -```bash -export DASK_SCHEDULER_ADDRESS=tcp://my-scheduler:8786 -lc run -``` - -`lc run` notices the env var and connects rather than starting its -own scheduler. It does *not* tear the scheduler down on exit. - -## NERSC Perlmutter: site-specific notes - -!!! note "Setting up on Perlmutter for the first time?" - The [Install](install.md) page has NERSC-specific tabs for Python - (uv vs `module load python`, conda env storage) and lightcone-cli. - Come back here once `lc --version` works. - -### Storage: keep Snakemake state on `$SCRATCH` - -!!! danger "DVS silently ignores `flock()`" - `$HOME` and `/global/cfs/` are mounted on compute nodes via DVS, - which silently ignores `flock()`. Snakemake relies on `flock` for - locking, so its `.snakemake/` directory and Dask spill files - **must** live on Lustre (`$SCRATCH`), which honors `flock`. - Otherwise you get intermittent silent rule-rerun loops or hangs. - -`lc` redirects state automatically when it detects Perlmutter, so -this usually just works. To pin explicitly at project creation: - -```bash -lc init your-analysis --scratch '$SCRATCH' # kept verbatim, expanded at run time -``` - -Or, after the fact, edit `/.lightcone/lightcone.yaml`: - -```yaml -scratch_root: $SCRATCH -``` - -!!! warning "12-week purge on `$SCRATCH`" - Perlmutter purges `$SCRATCH` on a rolling 12-week window. For - outputs you need to keep, copy or symlink to - `/global/cfs/cdirs//`. - -### Further reading - -- [NERSC interactive jobs](https://docs.nersc.gov/jobs/interactive/) - — `salloc` patterns and reservation queues -- [Perlmutter system overview](https://docs.nersc.gov/systems/perlmutter/) - — node types and partitions -- [NERSC queue policy](https://docs.nersc.gov/jobs/policy/) - — QoS options for GPU and CPU partitions - -## Troubleshooting - -- `dask CLI is not on PATH inside the SLURM allocation`. Install - `lightcone-cli` into the venv that your sbatch script activates; - `dask` ships with `distributed`, which is a transitive dep. -- Workers never register. Usually means the SLURM node hostnames - aren't resolvable from each other; check `SLURMD_NODENAME` / - `gethostname()` and confirm the workers can reach the scheduler. -- Image not found on compute nodes. Re-run `lc build` on the login - node — the migrate step is the one that actually publishes the - image to the per-node cache. -- Snakemake locking errors or silent rule-rerun loops on Perlmutter. - `.snakemake/` ended up on DVS-mounted storage — set - `scratch_root: $SCRATCH` in the project's `.lightcone/lightcone.yaml`. -- `pip install` hangs or times out. Compute nodes have no public - internet — always install from a login node. -- `PermissionError` reading another user's symlinked `results/`. - Cross-user scratch path without group ACLs — request access from - the data owner, or copy the manifests into your own scratch. - -For the wiring detail, see -[engine/dask_cluster](../api/dask_cluster.md) in the maintainer docs. diff --git a/docs/user/environment.md b/docs/user/environment.md new file mode 100644 index 00000000..57b78aeb --- /dev/null +++ b/docs/user/environment.md @@ -0,0 +1,130 @@ +# The environment + +The locked environment **is** the execution environment. A project is +`pyproject.toml` + `uv.lock` + `.python-version`; everything a recipe +may use is declared there (or in `astra.yaml` as data inputs), and the +sandbox makes that declaration mechanical rather than aspirational. + +## Two modes, one substrate + +Mode is **derived, never configured**: + +- **direct** (the default): the environment lives in the project tree + (`.venv`), no image is ever built, no container runtime is needed. +- **containerized**: triggered solely by declaring a system layer in + `pyproject.toml`. From then on the generated image is the execution + world — recipes, workers, and probes all run from its baked + environment. + +`lc status` always tells you which mode you are in and why. + +## The sandbox + +Every recipe (and every `lc run` probe) executes inside an OS sandbox +restricted to its declared set: + +- **writes**: its own `results///` directory, scratch, + and `/tmp` — sibling outputs, manifests, and `astra.yaml` are + mechanically protected from a misbehaving recipe; +- **reads**: the project tree, declared inputs, and the OS baseline; +- **executes**: the locked environment's binaries plus a small, + versioned set of shell utilities. + +On Linux this is [Landlock](https://landlock.io/) (kernel ≥ 5.13, +unprivileged, nothing to install); on macOS, Seatbelt. Each output's +manifest records the enforcement that actually ran — mechanism, file +scope, network posture — in its `hermeticity` field. If no mechanism is +available the run proceeds *and says so*; nothing is ever silently +unsandboxed. `--require-sandbox` turns that into a refusal. + +A recipe that legitimately writes intermediates elsewhere in the tree +declares it, in the repo: + +```toml +[tool.lightcone.sandbox] +writable-project = ["my_output"] +``` + +Its manifest then honestly records `fs: project-rw`. + +## The denial message + +When the sandbox blocks something, the error is the interface: + +```text +blocked by lc sandbox: cannot execute /usr/bin/latex — +not part of the declared environment. + + if this is a tool the recipe needs, declare it in the system layer: + [tool.lightcone.image] + system-packages = ["texlive-latex-base"] + note: this containerizes the project — podman required — and + re-stages all materialized outputs. + + if this is a data file, declare it as an input in astra.yaml: … + + diagnostics: lc run --sandbox-debug · lc run --no-sandbox · lc status +``` + +Both remedies are always shown; the escape hatches stay in the +diagnostics trailer. `lc run --sandbox-debug` opens a shell *inside* +the sandbox to poke at what a recipe can see. + +## The container hatch + +When a dependency genuinely cannot come from PyPI — R, Julia, TeX, +compilers, a system library a locked package links against — declare it: + +```toml +[tool.lightcone.image] +system-packages = ["r-base-core", "libhdf5-dev"] +``` + +That one table is the entire surface, and its presence is the +escalation. `lc` renders the locked environment *plus* the declared +system layer into a content-addressed image: + +- The image is **generated, never authored** — there is no Containerfile + to write. (`Containerfile.extra` exists as a bounded escape for build + steps beyond apt; it is content-hashed into the identity.) +- The image is a *cache of the lock plus the system layer*: project + code never enters an image, so **code edits never trigger a build**. + Environment edits rebuild — exactly when a rebuild is meaningful. +- Execution is **digest-pinned**: the tag is a pure function of the + repo plus the engine, the build records the produced digest, and + every run asserts it. +- A custom base (e.g. a CUDA userland) is supported, digest-pinned: + `base = "nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:…"`. + Tag-only refs are refused — the identity must be a function of the + repo, not of registry state. + +Check PyPI first: h5py wheels bundle libhdf5, NVIDIA ships CUDA wheels, +BLAS rides inside numpy. The hatch is for what genuinely has no wheel. + +Dependency verbs never change: `uv add` runs on the host, bare, in both +modes (add `--no-sync` in containerized projects — the host `.venv` is +inert there). The next `lc materialize` picks changes up through the +ordinary rebuild path; `lc run` never builds (it points you at +`lc build`). + +Escalation is reversible: delete the table and the project returns to +direct mode. Either direction is an environment edit — every +materialized output goes stale, and `lc` says so up front: + +```text +environment changed: 14 materialized output(s) are now stale +``` + +## What the manifest records + +Every output's `.lightcone-manifest.json` carries the environment +identity (`env_version` — lock, interpreter pin, install settings, +system layer), the runtime attestation (platform, interpreter build, uv +version, GPU driver, threading knobs), the container image tag+digest +and the system layer's package snapshot hash when one ran, and the +`hermeticity` record. `lc verify` additionally surfaces outputs that +were produced unsandboxed or from a dirty git tree. + +The claim is **pinned environment identity, never bit-identical +outputs** — you always know exactly what an output was computed with, +and exactly what it would take to recompute it. diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index d16c6d79..be903ac3 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -1,292 +1,131 @@ -# Getting Started +# Getting started -Let's go from nothing on your disk to a working, reproducible analysis. -You can read this top to bottom without running anything, or follow along — -every command is copy-paste ready. +This walkthrough takes an analysis from empty directory to verified, +provenance-tracked outputs. -**What you'll build:** a small two-output analysis that fits a linear model on -a public dataset and sweeps one methodological decision (whether to standardize -features). The result is two universes, `baseline` and `raw`, each with its -own `r2` metric and `fit_plot` figure — a clean comparison ready for a paper -figure. - -Make sure you've finished the [install](install.md) first. - -## 1. Create a project +## 1. Scaffold a project ```bash -lc init r2-decision-demo -cd r2-decision-demo -``` - -`lc init` converges the directory to a small, opinionated layout and -stops; it doesn't ask any questions, and it's idempotent — re-running -it later only fills in whatever is missing. - -``` -r2-decision-demo/ -├── astra.yaml # the spec — this is where everything lives -├── .gitignore -├── .git # initialized git repository (skip with --no-git) -├── .venv/ # Python virtual env with the analysis dependencies (skip with --no-venv) -├── .lightcone/ # internal scratchpad — don't edit by hand -├── Containerfile # build instructions for the project container -├── requirements.txt # software dependencies -├── myst.yml # MyST report configuration -├── index.md # template report that references the spec -├── universes/ -│ └── baseline.yaml # one universe, built from decision defaults -└── results/ - └── README.md # outputs materialize here via `lc run` +lc init my-analysis +cd my-analysis ``` -The file you'll actually work in: +`lc init` converges the directory into an ASTRA project (it is +idempotent — safe to re-run any time, it never overwrites files you +own). The scaffold: -**`astra.yaml`** — the single source of truth for your analysis. Inputs, -outputs, methodological decisions, recipes. Everything else lightcone-cli does -is downstream of this file. The boilerplate from `lc init` has one example -output and an example decision — enough to run `lc run` and see something -materialize, but not yet a real analysis. +| File | Role | +|---|---| +| `astra.yaml` | The analysis specification — inputs, outputs, recipes, decisions | +| `pyproject.toml` + `uv.lock` + `.python-version` | The environment — the *only* place dependencies live | +| `.venv/` | The materialized environment (derived from the lock; never edited by hand) | +| `universes/baseline.yaml` | The default decision universe | +| `results/` | Where outputs materialize, one directory per universe and output | +| `AGENTS.md` | Working notes for AI agents (the boundary rules below) | +| `myst.yml` + `index.md` | A template report that references the analysis by path | -ASTRA specs are plain YAML, designed to be easy for both humans and AI -assistants to write. In this guide you'll write one by hand — it's short. +## 2. Add dependencies -## 2. Write the spec - -Open `astra.yaml` and replace the boilerplate with our analysis: a linear -regression on sklearn's bundled diabetes dataset, with one decision — whether -to standardize features before fitting. - -```yaml -version: "0.0.13" # ASTRA spec version — keep what the scaffold wrote -name: "R² with and without feature standardization" -description: "Linear regression on the diabetes dataset, sweeping the standardization choice." -container: Containerfile - -inputs: [] # the diabetes dataset ships with scikit-learn - -decisions: - standardize: - label: "Feature standardization" - rationale: "Standardizing changes coefficient scales and can shift R² for ridge-like models." - default: standardized - options: - standardized: { label: "StandardScaler before fit" } - raw: { label: "No preprocessing" } - -outputs: - - id: r2 - type: metric - description: "Coefficient of determination on the test split." - decisions: [standardize] - recipe: - command: python src/fit.py --standardize {decisions.standardize} --output {output} - - id: fit_plot - type: figure - description: "Predicted vs true scatter." - inputs: [r2] - recipe: - command: python src/plot.py --r2_dir {inputs.r2} --output {output} -``` - -A few things to notice: - -- Each output declares what it depends on: `r2` depends on the - `standardize` decision, `fit_plot` depends on the sibling output `r2`. -- Recipes reference those dependencies through placeholders — - `{decisions.standardize}`, `{inputs.r2}`, `{output}` — which `lc run` - expands at execution time. `{output}` is the output's own results - directory. -- The decision's options aren't hardcoded anywhere in code; the scripts - will take them as command-line arguments. - -Check the spec is well-formed: +Dependencies are managed with uv, and only with uv: ```bash -astra validate astra.yaml -``` - -(`astra` is the spec-side CLI; it ships with `astra-tools`, a dependency of -lightcone-cli.) - -## 3. Write the scripts - -Two short scripts, in a `src/` directory (`mkdir src` — the scaffold -doesn't create it; where code lives is your choice, the recipes above -just happen to point there). First `src/fit.py` — fits the model, -writes the R² metric and the test-set predictions: - -```python -import argparse -import json -from pathlib import Path - -import numpy as np -from sklearn.datasets import load_diabetes -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler - -parser = argparse.ArgumentParser() -parser.add_argument("--standardize", choices=["standardized", "raw"], required=True) -parser.add_argument("--output", required=True) -args = parser.parse_args() - -X, y = load_diabetes(return_X_y=True) -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.25, random_state=0 -) -if args.standardize == "standardized": - scaler = StandardScaler().fit(X_train) - X_train, X_test = scaler.transform(X_train), scaler.transform(X_test) - -model = LinearRegression().fit(X_train, y_train) - -out = Path(args.output) -out.mkdir(parents=True, exist_ok=True) -(out / "r2.json").write_text(json.dumps({"r2": model.score(X_test, y_test)})) -np.savez(out / "predictions.npz", y_true=y_test, y_pred=model.predict(X_test)) +uv add numpy astropy matplotlib ``` -Then `src/plot.py` — reads the upstream output directory, makes the figure: - -```python -import argparse -import json -from pathlib import Path +This edits `pyproject.toml`, updates `uv.lock`, and syncs `.venv`. The +lock is the environment's identity: every output's manifest records +which lock it was produced under, and changing the lock marks every +materialized output stale — visibly, at decision time. -import matplotlib +!!! tip "The boundary rule" + A `ModuleNotFoundError` under `lc run` or `lc materialize` always + means the same thing: add the package with `uv add`. Never install + into another environment by hand. -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import numpy as np +## 3. Describe the analysis -parser = argparse.ArgumentParser() -parser.add_argument("--r2_dir", required=True) -parser.add_argument("--output", required=True) -args = parser.parse_args() - -r2_dir = Path(args.r2_dir) -r2 = json.loads((r2_dir / "r2.json").read_text())["r2"] -data = np.load(r2_dir / "predictions.npz") - -fig, ax = plt.subplots() -ax.scatter(data["y_true"], data["y_pred"], s=12) -ax.set_xlabel("true") -ax.set_ylabel("predicted") -ax.set_title(f"R² = {r2:.3f}") - -out = Path(args.output) -out.mkdir(parents=True, exist_ok=True) -fig.savefig(out / "fit_plot.png", dpi=150) -``` - -Finally, add the dependencies to `requirements.txt`: - -```text -scikit-learn -matplotlib -``` - -The Containerfile installs `requirements.txt` into the project image, so -that's all it takes — `lc run` rebuilds the image automatically when the -dependency files change. (If you're running without a container runtime, -install the same packages into `.venv` instead.) - -## 4. Add the second universe - -`lc init` scaffolded `universes/baseline.yaml`. Point it at our decision's -default: +Edit `astra.yaml`. A recipe's `command` is a template over the declared +inputs, decisions, and output directory: ```yaml -id: baseline -description: "Standardized features (the default)." -decisions: - standardize: standardized -``` +outputs: + - id: hubble_fit + type: metric + inputs: [supernovae] + decisions: [cosmology] + recipe: + command: > + python src/fit.py --data {inputs.supernovae} + --model {decisions.cosmology} --out {output} -And add the sweep — `universes/raw.yaml`: +inputs: + - id: supernovae + type: data + source: data/union2.1.txt -```yaml -id: raw -description: "No preprocessing before the fit." decisions: - standardize: raw + cosmology: + label: "Cosmological model" + default: flat_lcdm + options: + flat_lcdm: {label: "Flat ΛCDM"} + wcdm: {label: "wCDM"} ``` -Each universe is one complete selection of decision values; its results -materialize to `results///`. +## 4. Probe interactively -## 5. Run it +`lc run ` runs any command inside **exactly** the recipe +environment — same interpreter, same locked packages, same sandbox: ```bash -lc run +lc run python -c "import astropy; print(astropy.__version__)" +lc run # opens a shell in the recipe environment (sandboxed) ``` -`lc run` materializes every universe it finds under `universes/`. To run just -one, or just one output: - -```bash -lc run --universe baseline -lc run r2 -``` +Probes never materialize outputs. That's the next verb. -Then check where things stand: +## 5. Materialize ```bash -lc status +lc materialize # everything, all universes +lc materialize hubble_fit # one output +lc materialize -u baseline # one universe ``` -Expected output: +Each recipe runs inside a sandbox restricted to its declared set (see +[The Environment](environment.md)), and each output directory gains a +`.lightcone-manifest.json` recording exactly how it was produced: +recipe, environment identity, decisions, input hashes, content hash, +and the enforcement it ran under. -``` -Universe baseline - ✓ ok r2 - ✓ ok fit_plot +## 6. Inspect and verify -Universe raw - ✓ ok r2 - ✓ ok fit_plot +```bash +lc status # what's materialized / stale / missing, plus the env header +lc verify # recompute hashes; walk the provenance chain ``` -Your comparison is on disk: `results/baseline/r2/r2.json` vs -`results/raw/r2/r2.json`, with a figure next to each. +`lc status` never runs anything — it reads manifests, offline. A fresh +clone of a finished project reports its state without any setup. -If a recipe fails, `lc run` surfaces the error; fix the script or the spec -and rerun — only the affected outputs re-execute. Commit as you go so your -`git log` is a clean record of the build. - -## 6. Verify integrity +## 7. Publish ```bash -lc verify +lc export wrroc -o bundle.zip --zip ``` -This recomputes data hashes for every output and walks the input chain back to -declare whether anything has been tampered with since materialization. Useful -pre-publication, when archiving a project, or any time you want a stronger -guarantee than `lc status`. - -## What just happened - -- `astra.yaml` was the only place your analysis was *described* — inputs, - outputs, the decision, and the recipes all live there. -- The scripts take decision values as plain command-line arguments, so - nothing methodological is hardcoded. -- `lc run` generated `.lightcone/Snakefile` from your spec, dispatched each - rule through Snakemake, and wrote a per-output sidecar manifest recording the - recipe, container image, decisions, input hashes, and output hash. -- `lc status` and `lc verify` rely on those manifests — they don't re-execute - anything; they just check. +emits a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) +bundle (manifests, spec, lockfile, data) ready for Zenodo or +WorkflowHub. -If your laptop dies tomorrow and you `git clone` the repo on a fresh machine -and run `lc run`, you'll get bit-identical results. +## The four verbs -## Where to next +| Verb | Does | +|---|---| +| `lc run ` | probes — arbitrary commands in the recipe environment | +| `lc materialize` | executes — produces outputs with manifests | +| `lc status` | reports — offline, manifest-driven | +| `lc verify` | audits — recomputes the provenance chain | -- [Running on a Cluster](cluster.md) — take the same project to SLURM. -- [Troubleshooting](troubleshooting.md) — when something goes sideways. -- [Glossary](glossary.md) — terms like universe, decision, and manifest in - plain language. -- The [ASTRA docs](https://astra-spec.org/latest/) — the full spec: - sub-analyses, prior insights, findings, and evidence. +Outputs are materialized, not run: `lc run ` is an error +with a pointer to `lc materialize `. diff --git a/docs/user/glossary.md b/docs/user/glossary.md index aa0060ea..683176d8 100644 --- a/docs/user/glossary.md +++ b/docs/user/glossary.md @@ -1,175 +1,68 @@ # Glossary -The terms you'll see all over the docs and the `lc` command output, in -plain language. - -## ASTRA - -**A**gentic **S**chema for **T**ransparent **R**esearch **A**nalysis. -The schema lightcone-cli is built around. ASTRA's job is to capture an -analysis's inputs, outputs, and methodological decisions in a single -file (`astra.yaml`); lightcone-cli's job is to execute that spec -reproducibly. ASTRA ships separately as the `astra-tools` package and -the `astra` CLI handles the spec itself (validation, paper management, -evidence verification). - -## astra.yaml - -Your project's spec file. The single source of truth — every input, -output, recipe, and decision is declared here. Sub-analyses can be -nested via `analyses:` references. - -## Recipe - -A short shell or Python command that produces an output. Lives inside -an output's `recipe:` block in `astra.yaml`. Outputs declare which -sibling outputs they depend on, and the recipe references them through -placeholders: - -```yaml -outputs: - - id: r2 - recipe: - command: python src/fit.py --output {output} - - id: fit_plot - inputs: [r2] - recipe: - command: python src/plot.py --r2_dir {inputs.r2} --output {output} -``` - -## Decision - -A methodological choice with multiple defensible options (e.g. -"standardize features?", "what outlier threshold?"). Decisions live -in the `decisions:` section of `astra.yaml` along with their `default`, -their `options`, and their `rationale`. - -## Universe - -One specific selection of decision values. Universes live as YAML -files in `universes/` (e.g. `universes/baseline.yaml`, -`universes/permissive.yaml`). Each universe materializes its results -to its own directory: `results///`. - -If your spec has no universes, `lc run` materializes against a -universe called `"default"` with all decisions at their declared -defaults. - -## Sub-analysis - -A nested ASTRA analysis with its own inputs, outputs, and decisions, -referenced from a parent's `analyses:` section. The full tree shares -one set of universes; sub-analyses can reference parent decisions -with `from:` references. Sub-analyses are useful when an analysis has -genuinely different stages (training vs. inference, fit vs. evaluate); -keep things in one analysis when they share the same product. - -## Manifest - -The per-output sidecar JSON file -(`/.lightcone-manifest.json`) that records what produced -the output and what's inside it. Fields include `code_version`, -`data_version`, `container_image`, `recipe`, `decisions`, -`input_versions`, `git_sha`, `host`, `lc_version`, and a few more. -Manifests are written atomically by `lc run` and read by `lc status` -and `lc verify`. - -## code_version - -A SHA-256 over `(recipe + container_image + decisions)`. The -fingerprint of "what does this rule do?" When it drifts, downstream -outputs go `stale` in `lc status`. - -## data_version - -A SHA-256 over the contents of an output directory (excluding the -manifest itself). The fingerprint of "what bytes were produced?" -`lc verify` recomputes this and compares to the recorded value to -catch tampering. - -## input_versions - -Inside a manifest, a dict mapping each declared input id to its -version: the upstream output's `data_version` when the input is -another materialized output, or an `mtime-size`/`sha256` -fingerprint when the input is an external file. This is the chain -`lc verify` walks back through. - -## Container - -A Docker / Podman / podman-hpc image used to execute a recipe in -isolation. Declared at the analysis level (`container: Containerfile`) -or per-recipe (`recipe: { container: python:3.12-slim }`). Recipe-level -overrides win. - -## Containerfile - -A Dockerfile by another name (the syntax is identical). lightcone-cli -calls them Containerfiles to make clear they work with podman as well -as docker. - -## Image tag - -The string the runtime uses to identify a built image. lightcone-cli -generates content-addressed tags for Containerfile builds: -`lc--`. The hash covers the Containerfile and -your dependency files, so tags only change when the inputs to the -build change. - -## Runtime - -The OCI tool that actually executes containers: `docker`, `podman`, -or `podman-hpc`. Set in `~/.lightcone/config.yaml` under -`container.runtime`. `auto` picks the first usable; `none` opts out -(runs recipes directly on the host). - -## Snakemake - -The workflow engine `lc run` shells out to. You don't need to learn -Snakemake to use lightcone-cli — the Snakefile at `.lightcone/Snakefile` -is auto-generated from your `astra.yaml`. If you're curious, peek at -it; just don't edit it (your changes will get overwritten on the -next `lc run`). - -## Dask - -The distributed scheduler `lc run` dispatches jobs through. On a -laptop it's a `LocalCluster` sized to your machine; inside a SLURM -allocation it's an in-process scheduler with one `dask worker` per -node launched via `srun`. - -## Prior insight - -A piece of evidence from the literature that informs a decision. -Lives in the `prior_insights:` section of `astra.yaml`. Each insight -has a `claim`, one or more `evidence` entries with verbatim quotes, -and a list of decision options it supports. Quotes are -machine-verified against the source PDF. - -## Finding - -A conclusion drawn *from* the analysis (as opposed to a prior -insight, which comes *into* the analysis). Findings live in the -`findings:` section, can cite specific outputs as evidence, and act -as the bridge between materialized results and the eventual paper. - -## Status (`ok`, `stale`, `missing`, `alias`) - -The four labels `lc status` produces: - -- `ok` — manifest present, recomputed `code_version` matches. -- `stale` — manifest present but `code_version` drifted. -- `missing` — no manifest at the expected output directory. -- `alias` — output declared without a recipe; just a reference to - another output. - -## Failure kinds (`tampered_data`, `broken_chain`, `missing_manifest`) - -The three labels `lc verify` produces when something's wrong: - -- `tampered_data` — bytes on disk no longer match recorded - `data_version`. -- `broken_chain` — recorded `input_versions` references an upstream - whose `data_version` drifted. -- `missing_manifest` — output directory exists but the manifest is - missing or unparseable. +**ASTRA** — the specification language (`astra.yaml`): inputs, outputs, +recipes, decisions, universes. lightcone-cli is its execution layer. +ASTRA carries analysis structure only; the environment lives in the uv +project files. + +**direct mode** — the default execution mode: the locked environment +lives in the project tree (`.venv`), recipes run on the host inside the +OS sandbox, no image exists. + +**containerized mode** — entered by declaring `[tool.lightcone.image]` +(or a `Containerfile.extra`): the generated, content-addressed image +becomes the execution world for engine, workers, recipes, and probes. + +**env_version** — the environment identity: a hash of `uv.lock`, +`.python-version`, the uv install settings, and the declared system +layer. Part of every output's `code_version`; an environment edit +stales every materialized output, visibly. + +**code_version** — the identity of one output's materialization +semantics: recipe text, active decisions, `env_version`, and the +output's sandbox escalation. `lc status` compares it against manifests. + +**data_version** — the content hash of an output directory. `lc verify` +recomputes it to detect tampering. + +**manifest** — `.lightcone-manifest.json`, written beside every +materialized output: the versions above plus input hashes, git state, +runtime attestation, image identity, and the hermeticity record. The +provenance chain is manifests referencing manifests. + +**hermeticity** — the manifest field recording what enforcement a +recipe *actually* ran under: mechanism (`landlock`, `seatbelt`, +`podman+landlock`, `none`), file scope (`declared`, `project-rw`, +`open`), and network posture (`denied`, `allowed`, `unenforced`). + +**sandbox** — the OS enforcement (Landlock on Linux, Seatbelt on macOS) +that restricts each recipe to its declared set: own output dir +writable, project + declared inputs readable, locked environment plus a +versioned utility allowlist executable. + +**system layer** — apt packages (and optionally a digest-pinned base +image) declared in `[tool.lightcone.image]`; what flips a project into +containerized mode. + +**image tag (`lc-env-`)** — the content-addressed identity of the +generated image: a pure function of the rendered Containerfile, +`pyproject.toml`, and `uv.lock`. Code edits never move it. + +**probe (`lc run`)** — an arbitrary command run in byte-for-byte the +recipe environment (lock, interpreter, sandbox included), with writes +confined to the tmp scope. Probes never materialize outputs. + +**universe** — one assignment of options to the analysis's declared +decisions (`universes/.yaml`). Outputs materialize per universe +under `results///`. + +**decision** — a declared methodological choice with enumerated +options; the multiverse is the set of defensible option combinations. + +**blast radius** — the count of materialized outputs an environment +edit stales, printed at decision time: "environment changed: N +materialized output(s) are now stale". + +**pre-migration** — a manifest written by an earlier schema version. +Shown distinctly by `lc status`; `lc verify` still checks the hashes it +carries. diff --git a/docs/user/index.md b/docs/user/index.md index 7170e4a9..6acb30b9 100644 --- a/docs/user/index.md +++ b/docs/user/index.md @@ -1,62 +1,33 @@ -# Welcome to the user guide - -`lightcone-cli` is a small toolchain that turns a research question into -a reproducible analysis. You describe what you're trying to learn as a -precise specification — an `astra.yaml` file following the -[**ASTRA**][astra] schema — and the `lc` command line keeps the -resulting code, decisions, and outputs in sync. - -ASTRA specs are plain YAML, designed to be easy for both humans and AI -assistants to write. However the spec gets written, **you stay in charge -of the scientific choices** — every methodological decision is declared -in the open, and `lc` records exactly what produced every result. - -## What this guide covers - -- [Install](install.md) — get the `lc` command line running on your - machine or on a cluster. -- [Getting Started](getting-started.md) — create your first project, - run it end-to-end, and understand what each piece does. -- [Running on a Cluster](cluster.md) — taking your analysis to a SLURM - HPC system, including Perlmutter-specific notes. -- [Troubleshooting](troubleshooting.md) — common issues and how to - unstick them. -- [Glossary](glossary.md) — the terms that show up everywhere - (universe, decision, manifest, …) explained in plain language. - -## What you'll do, in three lines - -!!! tip "Quick start" - - === "uv" - ```bash - uv tool install lightcone-cli - lc init my-analysis && cd my-analysis - # describe your analysis in astra.yaml, then: - lc run - ``` - - === "pip" - ```bash - pip install lightcone-cli - lc init my-analysis && cd my-analysis - # describe your analysis in astra.yaml, then: - lc run - ``` - -That's the shortest possible path. The rest of the guide is the unhurried version. - -## What lightcone-cli is *not* - -- **A statistics package.** It runs your code; it doesn't compute - things itself. -- **A workflow language.** Recipes in `astra.yaml` are short shell or - Python commands, not a DSL. There's no learning curve beyond what's - in [Getting Started](getting-started.md). -- **An IDE.** `lc` is a command-line tool; write `astra.yaml` and your - analysis code with whatever editor or tooling you prefer. - -If you'd rather skim the design and architecture, the -[maintainer docs](../maintainer.md) are the other half of this site. - -[astra]: https://astra-spec.org/latest/ +# User guide + +lightcone-cli (`lc`) turns an `astra.yaml` analysis specification into +a tree of materialized, provenance-tracked outputs — with the +environment locked, the execution sandboxed, and every result carrying +a manifest that says exactly how it was made. + +- [Install](install.md) — uv is the only prerequisite. +- [Getting Started](getting-started.md) — from empty directory to + verified outputs. +- [The Environment](environment.md) — the locked environment, the + sandbox, and the container hatch. +- [Troubleshooting](troubleshooting.md) — the errors you may meet and + what they're telling you. +- [Glossary](glossary.md) — the vocabulary in one place. + +## The workflow at a glance + +!!! example "A complete session" + + ```bash + lc init my-analysis && cd my-analysis + uv add numpy astropy # dependencies: always uv + $EDITOR astra.yaml # describe the analysis + lc run python src/explore.py # probe in the recipe environment + lc materialize # produce outputs + manifests + lc status # what exists, what's stale + lc verify # audit the provenance chain + lc export wrroc -o out.zip --zip # publishable bundle + ``` + +Four verbs: **`lc run` probes, `lc materialize` executes, `lc status` +reports, `lc verify` audits.** diff --git a/docs/user/install.md b/docs/user/install.md index 583dda3c..216e8429 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -1,207 +1,58 @@ # Install -To get started on a lightcone project, you need two things on your machine: Python 3.11+ and the lightcone command line tool `lc`. -A container runtime is optional but recommended. +lightcone-cli has exactly one prerequisite: [uv](https://docs.astral.sh/uv/). +uv is the environment substrate — it manages the project's Python +interpreter, its locked dependencies, and lightcone-cli itself. -## 1. Python +## 1. uv -If you don't already have a recent Python - -=== "macOS" - ```bash - brew install python@3.12 - ``` - -=== "Linux" - Your package manager (`apt install python3.12`, etc.) or - [pyenv](https://github.com/pyenv/pyenv) - -=== "Windows" - [python.org](https://www.python.org/downloads/) or WSL - -=== "NERSC Perlmutter" - NERSC doesn't ship `uv`, but it installs into your home dir with a - single curl: - - ```bash - curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.12 - ``` - - Both `uv` and an isolated Python 3.12 land under `~/.local/`. - Make sure `~/.local/bin` is on your `PATH`. - - ??? note "Alternative: NERSC's `python` module" - `module load python` gives you a ready-to-use distribution with - `conda`, `pip`, and many scientific packages already installed: - - ```bash - module load python # NERSC Python (3.11+) - ``` - - Convenient, but the module is shared and read-only. For custom - packages, build a conda env on top: - - ```bash - conda create -n your-env-name python=3.11 -y - conda activate your-env-name - ``` - - This is NERSC's [recommended path for `pip install`](https://docs.nersc.gov/development/languages/python/nersc-python/) - when you need custom packages. - - !!! warning "Storage: 40 GB home quota" - Conda envs land under `~/.conda/envs/` by default. The - Perlmutter home quota is **40 GB**, which gets eaten quickly. - NERSC recommends `/global/common/software//` for - larger envs. If you want them on `$SCRATCH` (note: 12-week - purge), move and symlink: - - ```bash - conda deactivate - mv ~/.conda/envs/your-env-name $SCRATCH/conda-envs/ - ln -s $SCRATCH/conda-envs/your-env-name ~/.conda/envs/your-env-name - ``` - -!!! tip "Recommendation" - We highly recommend the use of [uv](https://docs.astral.sh/uv/) to manage Python installation and virtual environments. - - `uv` can be installed in a single commandline - - curl -LsSf https://astral.sh/uv/install.sh | sh - - and a subsequent version of Python +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` - uv python install 3.12 +(Or any of the [other installation methods](https://docs.astral.sh/uv/getting-started/installation/).) ## 2. lightcone-cli -The published name on PyPI is `lightcone-cli`; the command it provides -is `lc`. - -=== "uv" - ```bash - uv tool install lightcone-cli - ``` - -=== "pip" - ```bash - python -m pip install lightcone-cli - ``` - -=== "NERSC Perlmutter" - With `uv` (recommended — isolates `lc` under `~/.local/share/uv/tools/`): - - ```bash - uv tool install lightcone-cli - ``` - - With pip, the exact command depends on which Python you're using: - - ```bash - # NERSC python module - module load python - python -m pip install --user lightcone-cli # lands in ~/.local/bin/ - - # Conda env - conda activate your-env-name - python -m pip install lightcone-cli - ``` - - `astra-tools` is a transitive dependency — pulled in automatically. - - ??? note "From source (contributors only)" - ```bash - git clone https://github.com/LightconeResearch/lightcone-cli.git - uv pip install -e ./lightcone-cli - ``` - - To also hack on `astra-tools`: - - ```bash - git clone https://github.com/LightconeResearch/ASTRA.git - uv pip install -e ./ASTRA - ``` - -Get a confirmation of the proper installation by running - - lc --version # → lightcone-cli, version ... - -> **Note** Some people may have already set a personal shell alias `lc='ls --color'`. If that's you, installing lightcone-cli will shadow the alias — make sure to rebind it (e.g. `alias l='ls --color'`). - -## 3. Global configuration - -`~/.lightcone/config.yaml` is created automatically the first time you -run any `lc` command. No manual setup step is needed. The file starts -as: - -```yaml -container: - runtime: auto +```bash +uv tool install lightcone-cli ``` -`auto` detects whichever of `podman`, `docker`, or `podman-hpc` is on -your PATH (and skips docker if its daemon isn't running). Feel free to pin the runtime later by editing this file directly. - -## 4. (Optional) Docker or Podman +This puts the `lc` launcher on your PATH. The launcher is a thin shim: +each project locks its *own* copy of the engine (`lightcone-cli` is an +ordinary dependency in the project's `pyproject.toml`), and `lc` +delegates into it — so the engine version is pinned per experiment, in +the lock, like every other dependency. -If your analysis declares a `container:` (which it usually should — it -makes the result reproducible across machines), you need a container -runtime: +Verify: -- Local laptop: install [Podman](https://podman.io/) (rootless, no - daemon) or [Docker](https://docs.docker.com/get-docker/). -- HPC login node: see [Running on a Cluster](cluster.md). - -The `auto` mode picks whichever container runtime you have. If you don't -have either, you can still use `lc` — set `runtime: none` in -`~/.lightcone/config.yaml` and recipes will run on the host without -isolation. - -## Sanity check - - lc --help - lc init --help - -Both should print help text. If `lc` is shadowed by an `ls` alias, -unset it (`unalias lc`) or use the full path -(`$(which lc) --version`). - -## Updating - -=== "uv tool" - ```bash - uv tool upgrade lightcone-cli - ``` - -=== "pip" - ```bash - pip install -U lightcone-cli astra-tools - ``` - -=== "Source" - ```bash - cd path/to/lightcone-cli - git pull - uv pip install -e . # only needed if pyproject.toml changed - ``` - - Editable installs auto-follow source edits — switching branches or - pulling new commits is reflected immediately in `lc`. Re-install - only when `pyproject.toml` adds a new dependency. +```bash +lc --version +``` -## Uninstalling +## 3. (Only if you need it) podman + +Most projects never need a container. When a project declares system +dependencies uv cannot lock — R, TeX, compilers, system libraries — it +flips into **containerized mode** and needs rootless +[podman](https://podman.io/docs/installation): + +```bash +# Arch +sudo pacman -S podman +# Debian/Ubuntu +sudo apt install podman +# macOS (one-time Linux VM, ~minutes) +brew install podman +podman machine init && podman machine start +``` -=== "uv tool" - ```bash - uv tool uninstall lightcone-cli - ``` +You'll be told exactly when this becomes necessary — the sandbox denial +message names the step. Until then, there is nothing to install. -=== "pip" - ```bash - pip uninstall lightcone-cli - ``` +## Notes -!!! note "Keep your config?" - `~/.lightcone/config.yaml` survives the uninstall. Delete it too - if you want a clean slate. +- **Python**: you do not need a system Python. uv installs the exact + interpreter each project pins in `.python-version`. +- **No conda, no docker, no activation** — `lc ` is the whole + interface, from any directory inside a project. diff --git a/docs/user/troubleshooting.md b/docs/user/troubleshooting.md index b0867502..74b474d3 100644 --- a/docs/user/troubleshooting.md +++ b/docs/user/troubleshooting.md @@ -1,138 +1,79 @@ # Troubleshooting -Common issues and how to unstick them. Roughly ordered by how often -they come up. +## `ModuleNotFoundError` in a recipe or probe -## "No global configuration found." - -`~/.lightcone/config.yaml` is normally created automatically on first -use, but it may be missing if the home directory was unavailable or if -the file was deleted manually. Re-create it by hand: +The package is not in the project's lock. Fix the environment, never +the symptom: ```bash -mkdir -p ~/.lightcone -cat > ~/.lightcone/config.yaml <<'EOF' -container: - runtime: auto -EOF +uv add ``` -Or just run any `lc` command (e.g. `lc --version`) — the auto-creation -runs before every command. - -## "No astra.yaml found in current directory or any parent." - -You're outside an ASTRA project. Either: +(In a containerized project: `uv add --no-sync `.) Never +`pip install` into anything by hand — the sandbox only grants the +locked environment, so out-of-lock installs are invisible to recipes by +design. -```bash -cd path/to/your/project -``` +## `blocked by lc sandbox: cannot execute/read …` -or, if you're starting fresh: +The recipe touched something outside its declared set. The message +itself carries both remedies — declare the tool in +`[tool.lightcone.image]` (containerizes the project), or declare the +file as an input in `astra.yaml`. To investigate: ```bash -lc init my-analysis -cd my-analysis +lc run --sandbox-debug # a shell inside the sandbox ``` -`lc init` is idempotent — re-running it inside an existing project is -safe and just fills in anything missing (`lc init --check` tells you -whether it would change anything). - -## "lc: command not found" or `lc` prints a directory listing - -Two possibilities: - -1. The package isn't installed for your current Python. Check - `pip show lightcone-cli` (or `uv pip show lightcone-cli`). -2. Your shell has a personal alias `lc='ls --color'` shadowing the - real command. Run `type lc` to see; `unalias lc` to remove. - -## `lc run` warning: "No container runtime found on PATH" +## A recipe fails with a permissions/missing-file error but no denial message -You declared a container in `astra.yaml` but `auto` couldn't find any -of `docker`, `podman`, or `podman-hpc`. Two options: +Some programs swallow the underlying `PermissionError` and report +something else. Every failing sandboxed recipe prints the trailer +pointing at `lc run --sandbox-debug` — start there and try the exact +failing command inside the sandbox shell. -- **Install one.** Podman is the smallest install on Linux and macOS. -- **Opt out explicitly.** Edit `~/.lightcone/config.yaml`: - ```yaml - container: - runtime: none - ``` - This silences the warning, but then your manifests will record an - image that didn't actually run — fine for development, not fine for - archival. +## `the environment image lc-env-… is not built — run: lc build` -## `lc run` says "Workflow defines that rule … but no input" +`lc run` never builds images (a two-second probe must not silently +absorb a multi-minute build). Run `lc build` once; `lc materialize` +builds automatically and announces it. -This is Snakemake speak. It usually means: +## `environment changed: N materialized output(s) are now stale` -- A recipe declares `inputs: [foo]` but no other output produces - `foo`. Either the input is external (in which case it shouldn't be - in the recipe's `inputs:` list — recipes only chain to *sibling* - outputs), or there's a typo. -- Sub-analysis output ids that collide with root output ids — qualify - with `.`. +Not an error — the lock (or system layer) changed since those outputs +were produced, so their recorded environment no longer matches. +`lc materialize` re-runs exactly what's stale. -The fix is in `astra.yaml`. `astra validate astra.yaml` will catch -most typos. +## `environment changed mid-run` -## `lc status` shows everything `stale` after I just ran +The lock or `pyproject.toml` was edited while `lc materialize` was +running. Finish environment edits, then re-run — the double gate exists +so no manifest can claim an environment its recipe didn't run under. -Something in the spec changed in a way that affects `code_version`. -That hash covers recipe text, container image identifier, and -decisions. Common causes: +## `uv.lock contains unauditable dependencies` -- You edited a `Containerfile` or a dependency file (`requirements.txt`, - `pyproject.toml`). The image's content-addressed tag changed → - every recipe that uses it is now `stale`. -- You edited a recipe `command:`. Just rerun. -- You changed the default for a decision. +The lock references a path/directory/editable dependency (other than +the project's own package). Those bytes aren't pinned by the lock, so +provenance can't cover them — pin the dependency to a registry, or +vendor the files as declared inputs. -Re-running `lc run` will bring everything back to `ok`. +## `No astra.yaml found` -## `lc verify` fails with `tampered_data` +You're outside a project. `lc` discovers the project by walking up to +the nearest `astra.yaml`; run `lc init` to create one. -The bytes in an output directory no longer hash to the recorded -`data_version`. Most innocent cause: someone hand-edited a result -file. Most concerning: results were forged. +## macOS: `… lies outside the podman machine's shared directories` -If it was you, regenerate with `lc run --force `. If it -wasn't you, audit your shared filesystem. - -## `lc verify` fails with `broken_chain` - -A downstream output was materialized against an upstream version that -no longer exists. Usually caused by: - -- The upstream was rerun without rerunning the downstream. -- The upstream's output directory was edited by hand (which would also - trigger `tampered_data` on the upstream itself). - -Fix: `lc run` the downstream output. The chain will re-anchor. - -## I want to start the spec over - -Move `astra.yaml` aside (don't delete it — it's useful context about -what you tried), then write a fresh one: +The project (or a declared input) isn't visible inside podman's Linux +VM. The message names the fix: ```bash -mv astra.yaml astra.previous.yaml -$EDITOR astra.yaml +podman machine set --volume /path/shown/in/the/error +podman machine stop && podman machine start ``` -Re-running `lc init` afterwards is safe — it only fills in whatever is -missing and leaves the rest of the layout (`universes/`, `.lightcone/`) -as is. - -## Filing a bug - -Open an issue at -[github.com/LightconeResearch/lightcone-cli/issues](https://github.com/LightconeResearch/lightcone-cli/issues). -Include the output of `lc --version`, the command you ran, and the -error trace. - -## When all else fails +## `Another lc materialize holds the lock` -Run `lc verify` — it's the fastest way to know whether your problem -is provenance (real problem) or a transient build/run issue (rerun). +A concurrent run (or a crashed one whose process is still alive) holds +the project's run lock. Wait for it, or if you're certain it's gone, +delete the lockfile the message names. diff --git a/evals/prompt.md b/evals/prompt.md index 15a7a9db..81693e88 100644 --- a/evals/prompt.md +++ b/evals/prompt.md @@ -8,19 +8,25 @@ This project is driven by two CLIs — use them rather than improvising: `astra validate astra.yaml` checks it against the schema. If an `astra` skill or plugin is available in your environment, load it before reading or editing `astra.yaml` — it documents the full spec format. -- `lc` (lightcone-cli) is the execution layer, a thin shim over Snakemake: - - `lc run --universe baseline` materializes an output (and - anything upstream of it) by running the recipe commands declared in - `astra.yaml`. With no output ids it builds everything. It is +- `lc` (lightcone-cli) is the execution layer. Four verbs: + - `lc materialize --universe baseline` produces an output + (and anything upstream of it) by running the recipe commands declared + in `astra.yaml`. With no output ids it builds everything. It is idempotent: re-running only rebuilds what is stale or missing. + - `lc run ` probes: it runs an arbitrary command inside exactly + the recipe environment (same interpreter, same locked packages, same + sandbox). Use it to test imports or try a script before wiring it + into a recipe. Outputs are materialized, not run — `lc run + ` is an error. - `lc status --universe baseline` reports each output as `ok`, `stale`, or `missing`; `lc status --json` is the machine-readable form. + - `lc verify` audits the provenance chain. - Outputs land in `results/baseline//`, each with a `.lightcone-manifest.json` provenance manifest written by the engine. Files placed in `results/` by hand have no manifest and fail verification — never write there yourself. - - When `lc run` fails, read the error and the Snakemake log it points - to, fix the script or spec, and re-run. + - When `lc materialize` fails, read the error (and the log it points + to), fix the script or spec, and re-run. ## Recipe template grammar @@ -49,20 +55,21 @@ is how the engine orders the build. ## Environment -Recipes and your interactive shell run in two different environments — -keep them straight: +There is exactly one environment: the project's locked uv environment +(`pyproject.toml` + `uv.lock`). Recipes, probes, and your scripts all use +it. -- **Recipe commands run by `lc run`** may execute inside a container - built from the project's `Containerfile` + `requirements.txt` - (whenever `astra.yaml` declares a `container:` and a runtime is - available). Every package a recipe script imports must therefore be - listed in `requirements.txt` — add it there *before* running, and the - engine rebuilds the content-addressed image automatically. Host-side - installs never reach the container. -- **Your own shell commands** run on the host in an activated uv-managed - virtual environment with numpy, scipy, and matplotlib pre-installed. - For ad-hoc host tools use `uv pip install ` — plain `pip` is - not available in this venv. +- A `ModuleNotFoundError` always means the same thing: add the package + with `uv add ` — never install into any environment by hand, + and never use pip. +- Every recipe runs inside a sandbox restricted to its declared set: it + can write only its own output directory, read only the project and its + declared inputs, and execute only the locked environment plus basic + shell tools. If the sandbox blocks something, the error message itself + states the remedy (declare a data file as an input in `astra.yaml`, or + a system tool in `[tool.lightcone.image]`). +- `lc run --sandbox-debug` opens a shell inside the sandbox when you need + to see exactly what a recipe can see. ## Build loop @@ -73,9 +80,10 @@ that needs materializing: 1. Read the recipe's `command` to see what script and arguments it expects. 2. Write the script at the path the command names, parameterizing every decision via argparse — never hardcode option values. -3. Run `lc run --universe baseline` to materialize it through - the engine. -4. Commit progress as you go. +3. `uv add` any packages the script imports. +4. Run `lc materialize --universe baseline` to produce it + through the engine. +5. Commit progress as you go. Build iteratively from upstream outputs to downstream. `lc status --universe baseline` shows you what's `ok`, `stale`, or `missing` — you're diff --git a/evals/tasks/snae/astra.yaml b/evals/tasks/snae/astra.yaml index 3bdf0fb5..2d9db403 100644 --- a/evals/tasks/snae/astra.yaml +++ b/evals/tasks/snae/astra.yaml @@ -9,7 +9,6 @@ description: | using maximum-likelihood (MAP) point estimation. This provides best-fit cosmological parameters as a building block for a larger analysis. -container: Containerfile inputs: - id: union21 diff --git a/pyproject.toml b/pyproject.toml index 6e2d2532..5a755939 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,11 +36,6 @@ dependencies = [ "snakemake-interface-common>=1.14", "dask>=2024.1", "distributed>=2024.1", - # Dask Gateway client for JupyterHub/Kubernetes deployments. A - # normal dependency (not an extra) so `lc run` works out of the box - # on a hub and scaffolded project images inherit it via their - # lightcone-cli pin. - "dask-gateway>=2024.1", "rocrate>=0.11", ] @@ -98,13 +93,6 @@ module = ["dask.*", "distributed.*"] ignore_missing_imports = true follow_untyped_imports = true -[[tool.mypy.overrides]] -# Untyped and without explicit re-exports; skip so `from dask_gateway -# import Gateway` doesn't trip strict no_implicit_reexport. -module = ["dask_gateway", "dask_gateway.*"] -ignore_missing_imports = true -follow_imports = "skip" - [[tool.mypy.overrides]] module = ["rocrate.*"] ignore_missing_imports = true @@ -118,4 +106,6 @@ follow_untyped_imports = true testpaths = ["tests"] markers = [ "slow: tests that spin up real subsystems (dask cluster, etc.) — opt-in via -m slow", + "darwin: macOS-only enforcement smoke (runs in the macOS CI workflow)", + "podman: needs a working rootless podman — opt-in via -m podman", ] diff --git a/src/lightcone/_sandbox_exec.py b/src/lightcone/_sandbox_exec.py new file mode 100644 index 00000000..43375f17 --- /dev/null +++ b/src/lightcone/_sandbox_exec.py @@ -0,0 +1,97 @@ +"""The sandbox exec shim: ``python -m lightcone._sandbox_exec -- ARGV...`` + +Runs *between* fork and the recipe: applies the enforcement the parent +prepared, then execs the recipe argv. Deliberately stdlib-only with +zero lightcone imports — ``lightcone`` is a namespace package (no +``__init__``), so importing this module executes nothing else, keeping +the shim's footprint inside the exec path at effectively zero and +guaranteeing it can never drag engine code inside the sandbox setup. + +Contract (env, set by the parent's ``wrap_command``): + +* ``LC_SANDBOX_MODE`` — ``landlock`` | ``seatbelt`` | ``none``. +* ``LC_SANDBOX_FD`` — (landlock) the inherited ruleset FD. +* ``LC_SANDBOX_PROFILE`` — (seatbelt) path to the generated SBPL file. + +Exit code **97 is reserved for sandbox-setup failure** — the parent +attributes it to lc, never to the recipe, and it is the never-silent +guard: if the ruleset FD ever fails to survive to this point, the run +fails loudly instead of proceeding unsandboxed. + +The Landlock constants are duplicated from +``lightcone.engine.sandbox._landlock`` on purpose (no engine imports +here); a unit test pins the parity. +""" +from __future__ import annotations + +import ctypes +import ctypes.util +import os +import sys + +_SYS_LANDLOCK_RESTRICT_SELF = 446 +_PR_SET_NO_NEW_PRIVS = 38 + +#: Reserved exit code for sandbox-setup failure (never a recipe's). +SETUP_FAILURE_EXIT = 97 + +#: The wrap→shim env contract. Defined here (the shim must stay +#: stdlib-only) and imported by the engine's wrap layer. +SANDBOX_MODE_ENV = "LC_SANDBOX_MODE" +SANDBOX_FD_ENV = "LC_SANDBOX_FD" +SANDBOX_PROFILE_ENV = "LC_SANDBOX_PROFILE" + + +def _fail(message: str) -> None: + sys.stderr.write(f"lc sandbox setup failed: {message}\n") + sys.stderr.flush() + raise SystemExit(SETUP_FAILURE_EXIT) + + +def main() -> None: + argv = sys.argv[1:] + if argv and argv[0] == "--": + argv = argv[1:] + if not argv: + _fail("no command after --") + + mode = os.environ.pop(SANDBOX_MODE_ENV, None) + if mode == "landlock": + _restrict_landlock() + elif mode == "seatbelt": + _exec_seatbelt(argv) + return # unreachable + elif mode != "none": + _fail(f"unknown {SANDBOX_MODE_ENV} {mode!r}") + + os.execvp(argv[0], argv) + + +def _restrict_landlock() -> None: + fd_str = os.environ.pop(SANDBOX_FD_ENV, None) + if fd_str is None: + _fail(f"{SANDBOX_FD_ENV} not set") + try: + fd = int(fd_str) # type: ignore[arg-type] + os.fstat(fd) + except (ValueError, OSError): + _fail("ruleset fd did not survive to the shim") + return + libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0: + _fail(f"prctl(PR_SET_NO_NEW_PRIVS): errno {ctypes.get_errno()}") + if libc.syscall(_SYS_LANDLOCK_RESTRICT_SELF, fd, 0) != 0: + _fail(f"landlock_restrict_self: errno {ctypes.get_errno()}") + os.close(fd) + + +def _exec_seatbelt(argv: list[str]) -> None: + profile = os.environ.pop(SANDBOX_PROFILE_ENV, None) + if not profile or not os.path.isfile(profile): + _fail(f"seatbelt profile missing: {profile!r}") + return + os.execv("/usr/bin/sandbox-exec", ["sandbox-exec", "-f", profile, *argv]) + + +if __name__ == "__main__": + main() diff --git a/src/lightcone/cli/__init__.py b/src/lightcone/cli/__init__.py index ae164cc2..92957d61 100644 --- a/src/lightcone/cli/__init__.py +++ b/src/lightcone/cli/__init__.py @@ -11,6 +11,14 @@ def main() -> None: + import sys + + from lightcone.launcher import maybe_delegate + + # Execs the project-locked engine and never returns when it + # delegates; otherwise falls through to normal Click dispatch. + maybe_delegate(sys.argv[1:]) + from lightcone.cli.commands import main as _main _main() diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 51523b02..5634022e 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -1,20 +1,21 @@ """Command-line interface for lightcone-cli — the ASTRA execution layer. -The redesigned CLI is a thin shim over Snakemake. Provenance integrity -(per-output content-addressed manifests) is implemented in -:mod:`lightcone.engine.manifest`; ``lc run`` generates a Snakefile from -``astra.yaml`` and shells out to ``snakemake``. +The CLI is a thin shim over Snakemake. Provenance integrity (per-output +content-addressed manifests) is implemented in +:mod:`lightcone.engine.manifest`; the environment model (uv as the only +substrate, mode derived from the ``[tool.lightcone.image]`` hatch) in +:mod:`lightcone.engine.environment`. Commands: -- ``lc init`` — idempotently converge a project scaffold (spec, Containerfile, - gitignore, MyST report template, venv); ``--check``/``--json`` for agents. -- ``lc run`` — generate Snakefile and run snakemake. -- ``lc status`` — manifest-driven status walk (no Snakemake needed). -- ``lc verify`` — recompute hashes and validate the provenance chain. -- ``lc build`` — build containers from Containerfiles. - -The global config at ``~/.lightcone/config.yaml`` is auto-created with -defaults on first invocation if missing. +- ``lc init`` — idempotently converge a project scaffold; + ``--check``/``--json`` for agents. +- ``lc materialize`` — generate Snakefile and run snakemake. +- ``lc run`` — probe: run an arbitrary command in the recipe + environment (never materializes outputs). +- ``lc status`` — manifest-driven status walk (no Snakemake needed). +- ``lc verify`` — recompute hashes and validate the provenance chain. +- ``lc build`` — build the project's environment image + (containerized mode). """ from __future__ import annotations @@ -27,14 +28,14 @@ import subprocess import sys from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext from pathlib import Path import click import yaml from rich.console import Console -from lightcone.engine.container import ContainerBuildError +from lightcone.engine.environment import ProjectEnvironmentError +from lightcone.engine.image.errors import ImageError console = Console() logger = logging.getLogger(__name__) @@ -43,51 +44,27 @@ class _EngineErrorGroup(click.Group): """Render engine errors as clean CLI errors instead of tracebacks. - The engine raises :class:`ContainerBuildError` (and its subclass - ``CloudBuildError``) from many entry points — tag hashing, builds, - status walks. Translating once at the group boundary keeps every - command, present and future, from leaking a raw traceback; click - prints ``ClickException`` messages cleanly and exits 1. + The engine raises :class:`ProjectEnvironmentError` and + :class:`ImageError` from many entry points — environment loading, + identity hashing, builds, status walks. Translating once at the + group boundary keeps every command, present and future, from leaking + a raw traceback; click prints ``ClickException`` messages cleanly + and exits 1. """ def invoke(self, ctx: click.Context) -> object: try: return super().invoke(ctx) - except ContainerBuildError as e: + except (ProjectEnvironmentError, ImageError) as e: raise click.ClickException(str(e)) from e -def _config_path() -> Path: - return Path.home() / ".lightcone" / "config.yaml" - - -def _ensure_global_config() -> None: - """Create ``~/.lightcone/config.yaml`` with defaults if missing.""" - config = _config_path() - if config.exists(): - return - config.parent.mkdir(parents=True, exist_ok=True) - config.write_text( - yaml.safe_dump( - { - # Container runtime used by `lc build` and embedded in every - # recipe by `lc run`. ``auto`` picks the first of - # podman/docker/podman-hpc found on PATH (skipping docker if - # its daemon is unreachable); set explicitly to pin. ``none`` - # disables containerization entirely. - "container": {"runtime": "auto"}, - } - ) - ) - - @click.group(cls=_EngineErrorGroup) @click.version_option(package_name="lightcone-cli") @click.pass_context def main(ctx: click.Context) -> None: """lightcone-cli — execution layer for ASTRA projects.""" ctx.ensure_object(dict) - _ensure_global_config() # ============================================================================= @@ -97,14 +74,39 @@ def main(ctx: click.Context) -> None: def _project_root(start: Path | None = None) -> Path: """Walk up from cwd until we find ``astra.yaml``. Errors if absent.""" - p = (start or Path.cwd()).resolve() - for parent in [p, *p.parents]: - if (parent / "astra.yaml").is_file(): - return parent - raise click.ClickException( - "No astra.yaml found in current directory or any parent. " - "Run `lc init` to create one." - ) + from lightcone.engine.project import find_root + + root = find_root(start) + if root is None: + raise click.ClickException( + "No astra.yaml found in current directory or any parent. " + "Run `lc init` to create one." + ) + return root + + +def _load_env(project: Path): # type: ignore[no-untyped-def] + from lightcone.engine.environment import load_environment + + return load_environment(project) + + +def _assert_inside_image_for_containerized(mode: object) -> None: + """A containerized project's execution verbs run *inside* the image + — the launcher delegates there. Reaching this code on the host with + ``LC_DELEGATED=1`` set by hand would execute recipes outside the + declared environment and record provenance that misrepresents what + ran — refuse rather than pretend. + """ + from lightcone.engine.contract import in_container + from lightcone.engine.environment import Mode + + if mode is Mode.CONTAINERIZED and not in_container(): + raise click.ClickException( + "containerized projects execute inside the environment image " + "— invoke `lc` normally (the launcher delegates into the " + "image); do not set LC_DELEGATED by hand." + ) # ============================================================================= @@ -118,10 +120,21 @@ def _project_root(start: Path | None = None) -> Path: """ +def _run_uv(args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + """One seam for every uv invocation init makes (tests monkeypatch it).""" + return subprocess.run( + ["uv", *args], cwd=cwd, capture_output=True, text=True, check=False + ) + + @main.command() @click.argument("directory", type=click.Path(file_okay=False, path_type=Path), default=".") @click.option("--no-git", is_flag=True, help="Skip git init") -@click.option("--no-venv", is_flag=True, help="Skip Python venv creation") +@click.option( + "--no-sync", + is_flag=True, + help="Skip materializing the .venv (uv lock still runs).", +) @click.option( "--check", "check_only", @@ -144,14 +157,14 @@ def _project_root(start: Path | None = None) -> Path: type=str, help=( "Scratch root for snakemake state, dask spill, and run locks. " - "Overrides the site default. Shell expressions like $SCRATCH are " - "expanded at run time (kept verbatim in the project config)." + "Shell expressions like $SCRATCH are expanded at run time " + "(kept verbatim in the project config)." ), ) def init( directory: Path, no_git: bool, - no_venv: bool, + no_sync: bool, check_only: bool, as_json: bool, scratch_override: str | None, @@ -159,22 +172,36 @@ def init( """Converge DIRECTORY into an ASTRA project (idempotent). Safe to re-run at any time: creates whatever is missing, repairs the - pieces lightcone manages, and never overwrites files you own. - Problems it can see but must not fix (e.g. an unsupported directory - COPY in your Containerfile) are reported as warnings. A directory - that already holds an ``astra.yaml`` is adopted, not rejected. + pieces lightcone manages, and never overwrites files you own. A + directory that already holds an ``astra.yaml`` is adopted, not + rejected. The spec scaffold (``astra.yaml``, ``universes/baseline.yaml``) - follows the ``astra init`` boilerplate; on top of it sit the - lightcone pieces: ``Containerfile`` + ``requirements.txt``, - ``.gitignore`` entries, ``.lightcone/`` project state, a template - MyST report (``myst.yml`` + ``index.md``), and an optional venv. + follows the ``astra init`` boilerplate; on top of it sit the uv + project (``pyproject.toml`` with lightcone-cli locked in, + ``.python-version``, ``uv.lock``, ``.venv``), the agent notes + stanza (AGENTS.md), ``.gitignore`` entries, ``.lightcone/`` project + state, and a template MyST report (``myst.yml`` + ``index.md``). """ - from lightcone.engine.site_registry import detect_current_site - directory = directory.resolve() write = not check_only + # Refusal before any write: an authored Containerfile is the v3-era + # model. The user's own file operation is the consent to migrate — + # no flag can substitute for it. + if (directory / "Containerfile").is_file(): + raise click.ClickException( + f"{directory}/Containerfile: v6 generates images from the " + "lock — delete or rename it, then re-run `lc init`. Declare " + "system dependencies in [tool.lightcone.image] instead." + ) + + if shutil.which("uv") is None: + raise click.ClickException( + "uv is required (the environment substrate). Install it: " + "https://docs.astral.sh/uv/getting-started/installation/" + ) + report: dict[str, list[str]] = { "created": [], "repaired": [], @@ -242,67 +269,43 @@ def _scaffold_spec() -> None: # ``python src/main.py``); astra's own init creates the # directory, so the scaffold must too. (directory / "src").mkdir(exist_ok=True) - # Point the spec at our project-local Containerfile. The astra - # boilerplate ships a registry image so the scaffold is runnable - # as-is, but we want lightcone projects to build their own image - # so dependencies can evolve under content-addressed rebuilds. - # Rewrite the top-level ``container:`` line whatever image the - # boilerplate names, so astra bumping its default doesn't - # silently disable the rewrite. + # ASTRA carries only analysis structure — the environment lives + # in pyproject.toml + uv.lock. Strip any ``container:`` line the + # boilerplate ships. astra_yaml_path = directory / "astra.yaml" - rewritten = re.sub( - r"(?m)^container:.*$", - "container: Containerfile", - astra_yaml_path.read_text(), - count=1, + stripped = re.sub( + r"(?m)^container:.*\n?", "", astra_yaml_path.read_text(), count=1 ) - if "container: Containerfile" not in rewritten: - report["warnings"].append( - "astra.yaml: no top-level `container:` line found to point " - "at the Containerfile; set it manually." - ) - astra_yaml_path.write_text(rewritten) + astra_yaml_path.write_text(stripped) _converge("astra.yaml", (directory / "astra.yaml").exists(), _scaffold_spec) - # One scaffold everywhere — the Containerfile is agnostic to the - # execution environment. requirements.txt holds only the analysis - # dependencies; the execution stack (lightcone-cli, which carries - # snakemake, dask, distributed, dask-gateway) is a separate - # Containerfile layer so the same image can wrap recipes locally or - # run as a Dask Gateway worker pod on a hub, while the project venv - # stays free of it — `lc` lives outside the venv. Anything - # pod-specific (uid, mounts) is deployment configuration, not image - # content. - cf_path = directory / "Containerfile" - _converge_file( - "Containerfile", - cf_path, - _CONTAINERFILE_TEMPLATE.format(lc_requirement=_lightcone_requirement()), - ) - # Advisory: a Containerfile with directory COPY sources belongs to - # the user, so init won't edit it — but lc build / lc run will - # reject it, so say so now rather than at build time. - if cf_path.is_file(): - from lightcone.engine.container import directory_copy_sources - - if bad := directory_copy_sources(cf_path, directory): + # The uv project: pyproject.toml (virtual — no [build-system]) with + # the engine inside the experiment's lock, and the exact interpreter + # pin. Existing files are the user's: verified, never edited. + pyproject_path = directory / "pyproject.toml" + if not pyproject_path.exists(): + report["created"].append("pyproject.toml") + if write: + pyproject_path.write_text( + _PYPROJECT_TEMPLATE.format( + name=_project_name(directory), + lc_requirement=_lightcone_requirement(), + ) + ) + else: + report["unchanged"].append("pyproject.toml") + if "lightcone-cli" not in pyproject_path.read_text(): report["warnings"].append( - "Containerfile: COPY/ADD of a directory " - f"({', '.join(repr(s) for s in bad)}) is not supported and " - "lc build/run will fail. The image is an environment — " - "recipes run against the live project tree, so remove the " - "line(s)." + "pyproject.toml does not depend on lightcone-cli — the " + "engine should live inside the experiment's lock: " + "`uv add lightcone-cli`." ) - _converge_file( - "requirements.txt", - directory / "requirements.txt", - _REQUIREMENTS, - ) - # .gitignore: create with base + lightcone entries if absent; append - # the block once to a user-owned file (keyed on the "# lightcone-cli" - # marker). + from lightcone.engine.image.constants import DEFAULT_PYTHON + + _converge_file(".python-version", directory / ".python-version", f"{DEFAULT_PYTHON}\n") + _converge_file( ".gitignore", directory / ".gitignore", @@ -310,6 +313,8 @@ def _scaffold_spec() -> None: repair=_repair_gitignore, ) + _converge_file("AGENTS.md", directory / "AGENTS.md", _AGENTS_MD, repair=_repair_agents) + # .lightcone/ project state dir + lightcone.yaml. An explicit # --scratch converges the stored scratch_root; without it an # existing config is left alone. A file we can't parse is left @@ -320,7 +325,7 @@ def _scaffold_spec() -> None: report["created"].append(cfg_name) if write: cfg_path.parent.mkdir(exist_ok=True) - project_cfg: dict[str, object] = {"target": "local"} + project_cfg: dict[str, object] = {} if scratch_override: project_cfg["scratch_root"] = scratch_override cfg_path.write_text(yaml.safe_dump(project_cfg)) @@ -348,7 +353,7 @@ def _scaffold_spec() -> None: # contract — the placeholder directory alone is invisible in git # (empty + ignored), so the README is what actually tells a human # or agent opening the project where outputs land and that they - # must come from `lc run`, not be written by hand. + # must come from `lc materialize`, not be written by hand. results_dir = directory / "results" if results_dir.exists() and not results_dir.is_dir(): report["unchanged"].extend(["results/", "results/README.md"]) @@ -374,12 +379,32 @@ def _scaffold_spec() -> None: lambda: subprocess.run(["git", "init", "-q"], cwd=directory, check=False), ) - if not no_venv: - _converge( - ".venv", - (directory / ".venv").exists(), - lambda: _create_venv(directory, quiet=as_json), - ) + # Lock, then converge the environment. Failures surface — a silent + # broken lock would fail every later verb more confusingly. + def _lock() -> None: + proc = _run_uv(["lock", "--project", str(directory)], cwd=directory) + if proc.returncode != 0: + raise click.ClickException( + f"`uv lock` failed:\n{proc.stderr.strip()}" + ) + + _converge("uv.lock", (directory / "uv.lock").exists(), _lock) + + if not no_sync: + def _sync() -> None: + proc = _run_uv( + [ + "sync", "--locked", "--exact", "--compile-bytecode", + "--project", str(directory), + ], + cwd=directory, + ) + if proc.returncode != 0: + raise click.ClickException( + f"`uv sync` failed:\n{proc.stderr.strip()}" + ) + + _converge(".venv", (directory / ".venv").exists(), _sync) converged = not report["created"] and not report["repaired"] @@ -408,31 +433,19 @@ def _scaffold_spec() -> None: else: console.print(f"\n[green]Project converged at[/green] {directory}") - # Surface the resolved scratch root if a known site was detected — - # gives users early visibility into where lc run will keep its - # operational state (snakemake metadata, dask spill, cross-node - # locks). On NERSC this is critical: $HOME and CFS are mounted via - # DVS (no flock, slow small-file I/O), so lightcone keeps - # everything on $SCRATCH (Lustre). - site = detect_current_site() - if site: - scratch_expr = scratch_override or site.get("scratch_root") - if scratch_expr: - console.print(f"\n[dim]Detected site:[/dim] {site.display_name}") - console.print( - f"[dim]Scratch root for lc run:[/dim] [cyan]{scratch_expr}[/cyan] " - f"[dim](resolved at run time)[/dim]" - ) - # Next steps only make sense for a freshly scaffolded spec. if "astra.yaml" in report["created"]: console.print("\nNext steps:") console.print( f" • Go to the newly created directory [cyan]cd {directory}[/cyan]" ) + console.print( + " • Add analysis dependencies with [cyan]uv add[/cyan] " + "(e.g. [cyan]uv add numpy astropy[/cyan])" + ) console.print( " • Describe your analysis in [cyan]astra.yaml[/cyan], " - "then materialize it with [cyan]lc run[/cyan]" + "then materialize it with [cyan]lc materialize[/cyan]" ) console.print( " • Preview the report with [cyan]myst start[/cyan] " @@ -443,92 +456,42 @@ def _scaffold_spec() -> None: sys.exit(1) -def _create_venv(directory: Path, quiet: bool = False) -> None: - """Create ``.venv`` in ``directory`` with the analysis dependencies. - - Installs ``requirements.txt`` only — deliberately *not* - lightcone-cli. The venv exists to run the analysis code; ``lc`` - itself lives outside it (e.g. ``uv tool install lightcone-cli``), - and a second copy inside the venv would shadow it with whatever - version PyPI resolves. - """ +def _project_name(directory: Path) -> str: + """PEP 503-ish project name derived from the directory name.""" + name = re.sub(r"[^A-Za-z0-9._-]+", "-", directory.name).strip("-._").lower() + return name or "analysis" - def _status(msg: str) -> AbstractContextManager[object]: - return nullcontext() if quiet else console.status(msg) - if shutil.which("uv"): - with _status("[dim]Creating virtual environment…[/dim]"): - subprocess.run( - ["uv", "venv", "--python", "3.12", ".venv"], - cwd=directory, - check=False, - capture_output=True, - ) - with _status("[dim]Installing project requirements…[/dim]"): - subprocess.run( - [ - "uv", - "pip", - "install", - "--python", - ".venv/bin/python", - "-r", - "requirements.txt", - ], - cwd=directory, - check=False, - capture_output=True, - ) - else: - with _status("[dim]Creating virtual environment…[/dim]"): - subprocess.run( - ["python", "-m", "venv", ".venv"], - cwd=directory, - check=False, - capture_output=True, - ) - with _status("[dim]Installing project requirements…[/dim]"): - subprocess.run( - [ - ".venv/bin/python", - "-m", - "pip", - "install", - "-q", - "-r", - "requirements.txt", - ], - cwd=directory, - check=False, - capture_output=True, - ) - - -_CONTAINERFILE_TEMPLATE = """\ -FROM python:3.12-slim - -WORKDIR /app - -# Execution stack — lets this image run rules on any backend, including -# as a Dask Gateway worker pod. Kept out of requirements.txt so the -# project venv stays free of it (`lc` lives outside the venv), and -# installed first so this heavy layer stays cached across -# requirements.txt edits. -RUN pip install --no-cache-dir {lc_requirement} +def _lightcone_requirement() -> str: + """The lightcone-cli requirement pinned into the project scaffold. -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt + The engine lives *inside the experiment's lock* — pinned to the + version running ``lc init`` so driver and project stay in lockstep; + dev builds fall back to unpinned (their version isn't published). + """ + from importlib.metadata import PackageNotFoundError, version -# No COPY of the project source: recipes run against the live project -# tree (bind-mounted locally, shared filesystem on a hub), so the image -# is a pure environment — it only rebuilds when dependencies change, -# never on code edits. -""" + try: + v = version("lightcone-cli") + except PackageNotFoundError: + v = "" + return f"lightcone-cli=={v}" if v and "dev" not in v else "lightcone-cli" -_REQUIREMENTS = """\ -numpy -pandas +_PYPROJECT_TEMPLATE = """\ +[project] +name = "{name}" +version = "0.1.0" +requires-python = ">=3.12" +# Analysis dependencies: add with `uv add ` — never install +# into another environment by hand. The engine (lightcone-cli) is a +# normal locked dependency: the experiment pins its own execution layer. +dependencies = [ + "{lc_requirement}", +] + +[tool.uv] +required-version = ">=0.12" """ @@ -536,60 +499,18 @@ def _repair_gitignore(text: str) -> str | None: """Append the managed block once to a user-owned .gitignore. Keyed on the block's ``# lightcone-cli`` marker so re-runs never - duplicate it. This is `lc init`'s only repair hook: adoption of a - project that already has its own .gitignore. + duplicate it. """ if "# lightcone-cli" not in text: return text + _GITIGNORE_APPEND - # Legacy managed-block upgrades, applied line-by-line so everything - # else in the file stays user territory: - # * a bare ``results/`` rule ignores the whole directory, and git - # cannot re-include results/README.md beneath an excluded - # directory; - # * trailing-slash ``.snakemake/`` entries never match the symlink - # that ``.snakemake`` becomes under a scratch root. - slash_fixes = { - ".snakemake/": ".snakemake", - ".snakemake.legacy/": ".snakemake.legacy", - } - out: list[str] = [] - changed = False - for line in text.splitlines(): - stripped = line.strip() - if stripped == "results/": - out.append("results/*") - if "!results/README.md" not in text: - out.append("!results/README.md") - changed = True - elif stripped in slash_fixes: - out.append(slash_fixes[stripped]) - changed = True - else: - out.append(line) - if changed: - return "\n".join(out) + ("\n" if text.endswith("\n") else "") return None -def _lightcone_requirement() -> str: - """The lightcone-cli requirement pinned into the project image. - - The project image must be able to execute rules on any backend — - including as a Dask Gateway worker pod, where the dask worker and - the child snakemake run *inside* the image. lightcone-cli carries - that whole stack (snakemake, dask, distributed, dask-gateway) as - normal dependencies, so one requirement covers it. The pin mirrors - the version running ``lc init`` to keep driver and image in - lockstep; dev builds fall back to unpinned (their version isn't - published). - """ - from importlib.metadata import PackageNotFoundError, version - - try: - v = version("lightcone-cli") - except PackageNotFoundError: - v = "" - return f"lightcone-cli=={v}" if v and "dev" not in v else "lightcone-cli" +def _repair_agents(text: str) -> str | None: + """Append the lightcone stanza once to a user-owned AGENTS.md.""" + if "" not in text: + return text + "\n" + _AGENTS_STANZA + return None # Written when the project has no .gitignore of its own; mirrors the base @@ -598,7 +519,6 @@ def _lightcone_requirement() -> str: # ASTRA Analysis __pycache__/ *.py[cod] -.venv/ .ipynb_checkpoints/ .DS_Store """ @@ -612,8 +532,10 @@ def _lightcone_requirement() -> str: # directories. _GITIGNORE_APPEND = """ # lightcone-cli +.venv/ .lightcone/Snakefile .lightcone/snakefile-config.json +.lightcone/image/ .snakemake .snakemake.legacy results/* @@ -632,18 +554,40 @@ def _lightcone_requirement() -> str: results/// Each output directory carries a `.lightcone-manifest.json` sidecar -recording exactly how it was produced: recipe, container image, -decisions, input hashes, and the output's content hash. +recording exactly how it was produced: recipe, environment identity, +decisions, input hashes, the output's content hash, and the sandbox +enforcement it ran under. -- Produce or refresh outputs with `lc run` — never write files here by - hand. Hand-placed or edited files fail `lc verify` (the content hash - won't match, and a missing manifest forces a re-run). +- Produce or refresh outputs with `lc materialize` — never write files + here by hand. Hand-placed or edited files fail `lc verify` (the + content hash won't match, and a missing manifest forces a re-run). - `lc status` shows what is materialized, stale, or missing. - Everything in this directory except this README is git-ignored; outputs are reproducible from `astra.yaml`, not versioned. """ +_AGENTS_STANZA = """\ + +## Working in this lightcone project + +- The environment is `pyproject.toml` + `uv.lock` (+ `.python-version`). + A `ModuleNotFoundError` under `lc run`/`lc materialize` means: fix + `pyproject.toml` with `uv add ` — never install into another + environment by hand. +- `uv add` runs on the host, bare. In a containerized project + (`[tool.lightcone.image]` declared), add `--no-sync`; never + `lc run uv add`. +- The four verbs: `lc run ` probes (arbitrary commands in the + recipe environment), `lc materialize` executes outputs, `lc status` + reports, `lc verify` audits. +- Outputs are materialized, not run: `lc materialize `, + never `lc run `. +""" + +_AGENTS_MD = "# Agent notes\n\n" + _AGENTS_STANZA + + # The template report references the boilerplate ``astra.yaml`` elements by # path via the MySTRA plugin, so the ids used below must track the astra # boilerplate (``example_method``, ``main_result``). @@ -684,8 +628,8 @@ def _lightcone_requirement() -> str: ## Results -TODO: present the outputs. Once `lc run` has materialized results, pull -numbers in live, e.g.: +TODO: present the outputs. Once `lc materialize` has produced results, +pull numbers in live, e.g.: % The analysis yields {astra:value}`outputs.main_result`. @@ -694,41 +638,10 @@ def _lightcone_requirement() -> str: """ # ============================================================================= -# lc run +# lc materialize # ============================================================================= -def _abort_on_perlmutter_login() -> None: - """Stop-gap: refuse ``lc run`` on a Perlmutter login node. - - NERSC sets ``NERSC_HOST=perlmutter`` on every node; SLURM sets - ``SLURM_JOB_ID`` only inside an allocation. Their conjunction (NERSC - host + no allocation) unambiguously marks a login node, where shared - CPU and the absence of compute resources make a real run a bad idea. - - Bypassed when ``DASK_SCHEDULER_ADDRESS`` is set, matching the branch - in ``cluster_for_run``: if the user is targeting an external - scheduler the login-node CPU does not matter. - - Remove once proper site-backend gating exists. - """ - if os.environ.get("LIGHTCONE_ALLOW_LOGIN_NODE"): - return - if os.environ.get("NERSC_HOST") != "perlmutter": - return - if "SLURM_JOB_ID" in os.environ: - return - if os.environ.get("DASK_SCHEDULER_ADDRESS"): - return - raise click.ClickException( - "Refusing to run on a Perlmutter login node — compute work must " - "run inside a SLURM allocation.\n" - " Start one with, e.g.:\n" - " salloc -N 1 -C gpu -q interactive -t 1:00:00 -A \n" - " then re-run `lc run` from inside." - ) - - @main.command() @click.argument("outputs", nargs=-1) @click.option("--universe", "-u", default=None, help="Universe to materialize") @@ -740,25 +653,39 @@ def _abort_on_perlmutter_login() -> None: ) @click.option("--force", "-f", is_flag=True, help="Force re-materialization") @click.option("--verbose", "-v", is_flag=True, help="Show full executor output") -def run( +@click.option( + "--require-sandbox", + "require_sandbox", + is_flag=False, + flag_value="any", + default=None, + help=( + "Refuse to run recipes without a sandbox mechanism; " + "--require-sandbox=declared-fs additionally requires " + "declared-file scoping." + ), +) +@click.option( + "--no-sandbox", + is_flag=True, + help="Run recipes without the sandbox (recorded as unsandboxed).", +) +def materialize( outputs: tuple[str, ...], universe: str | None, jobs: int | None, rerun_triggers: str, force: bool, verbose: bool, + require_sandbox: str | None, + no_sandbox: bool, ) -> None: """Materialize outputs declared in astra.yaml. - Always dispatches through a Dask cluster: a ``LocalCluster`` on a - workstation, srun-launched workers inside a SLURM allocation, a - run-scoped Dask Gateway cluster on a JupyterHub deployment, or an - existing scheduler if ``DASK_SCHEDULER_ADDRESS`` is set. + Dispatches through a run-scoped Dask ``LocalCluster``. """ - _abort_on_perlmutter_login() - - from lightcone.engine.container import load_runtime - from lightcone.engine.dask_cluster import cluster_for_run, gateway_branch_active + from lightcone.engine.dask_cluster import cluster_for_run + from lightcone.engine.environment import scan_lock from lightcone.engine.scratch import ( RunLockBusyError, acquire_run_lock, @@ -767,80 +694,42 @@ def run( resolve_scratch_root, ) from lightcone.engine.snakefile import discover_universes, generate + from lightcone.engine.status import env_blast_radius project = _project_root() + env = _load_env(project) + _assert_inside_image_for_containerized(env.mode) universes = [universe] if universe else discover_universes(project) + # Blast radius: surfaced before anything runs, so an environment + # edit's cost is visible at decision time. + if (n_stale := env_blast_radius(project, universes=universes, env=env)) > 0: + console.print( + f"[yellow]environment changed:[/yellow] {n_stale} materialized " + "output(s) are now stale" + ) + scan = scan_lock(project) + if scan.non_default_groups: + console.print( + f"[dim]note: non-default dependency group(s) " + f"{', '.join(scan.non_default_groups)} are outside lc's guarantees[/dim]" + ) + # Resolve scratch and prepare per-run directories before anything # else. Snakemake's ``.snakemake/`` is redirected via symlink so its - # workflow lock and metadata land on a filesystem that honours - # ``flock`` (Lustre on NERSC) rather than DVS-mounted home/CFS where - # locks are silent no-ops. Dask spill and our cross-node stdout lock - # live alongside it. + # workflow lock and metadata land under the scratch root; dask spill + # and the run lock live alongside it. rundirs = prepare_run_dirs(project) ensure_snakemake_symlink(project, rundirs.snakemake_state) if verbose: console.print(f"[dim]Scratch root:[/dim] {resolve_scratch_root(project)}") - choice = load_runtime(project_path=project) - images = _ensure_images(project, runtime=choice.runtime) - snakefile_path, cfg_path = generate( - project, universes=universes, runtime=choice.runtime + snakefile_path, _cfg_path = generate( + project, universes=universes, env=env, scan=scan ) - # On the Gateway branch the cluster is created with one image — the - # worker pod is the container for every rule, so a spec declaring - # several distinct containers cannot be honoured per-rule. - worker_image: str | None = None - if gateway_branch_active(): - if len(images) > 1: - raise click.ClickException( - "This deployment runs recipes natively in worker pods, " - "which supports one container image per run; astra.yaml " - "declares several: " + ", ".join(images) + ". " - "Consolidate on a single Containerfile (or one shared " - "prebuilt image)." - ) - worker_image = images[0] if images else None - - # Provenance guard: when ``runtime: auto`` silently fell back to - # ``none`` and the spec declares any containers, the recipe will run - # on the host while the manifest's ``container_image`` field still - # records the declared image — i.e. a provenance lie. Warn loudly so - # the user installs a runtime, sets ``runtime: none`` explicitly, or - # removes the container declarations. - if choice.runtime == "none" and not choice.explicit: - cfg_data = json.loads(cfg_path.read_text()) - declared = sorted( - { - entry["container_image"] - for rule_entries in cfg_data.values() - for entry in rule_entries.values() - if entry.get("container_image") - } - ) - if declared: - console.print( - "[yellow]⚠ No container runtime found on PATH " - "(checked docker, podman, podman-hpc).[/yellow]\n" - " The following declared containers will be ignored:\n" - + "\n".join(f" [dim]•[/dim] {c}" for c in declared) - + "\n Recipes will run on the host without isolation, " - "but each manifest will still record\n" - " the declared [cyan]container_image[/cyan] — recorded " - "provenance will not match what executed.\n" - " Install [cyan]docker[/cyan], [cyan]podman[/cyan], or " - "[cyan]podman-hpc[/cyan], or set\n" - " [cyan]container: {runtime: none}[/cyan] in " - "[cyan]~/.lightcone/config.yaml[/cyan] to silence.\n" - ) - - targets: list[str] = [] - if outputs: - for o in outputs: - for u in universes: - targets.append(_target_for(project, o, u)) # If no specific targets, pass nothing → snakemake runs `rule all`. + targets = _targets_for(project, outputs, universes) if outputs else [] n = str(jobs or os.cpu_count() or 1) # Snakemake requires ``--cores`` to bound per-rule CPU; the dask @@ -859,26 +748,32 @@ def run( # Hold a project-level flock for the duration of the run. Acquiring # it also clears any stale snakemake lock left by a previously # crashed invocation — safe because we just proved we're alone on - # the project. Concurrent ``lc run`` on the same project bails - # cleanly rather than corrupting Snakemake state. + # the project. Concurrent ``lc materialize`` on the same project + # bails cleanly rather than corrupting Snakemake state. try: run_lock_cm = acquire_run_lock(rundirs) run_lock_cm.__enter__() except RunLockBusyError as e: raise click.ClickException(str(e)) + from lightcone.engine.runner import NO_SANDBOX_ENV, REQUIRE_SANDBOX_ENV + with cluster_for_run( verbose=verbose, local_directory=str(rundirs.dask_local), - worker_image=worker_image, - max_workers=int(n), ) as cluster_env: - env = {**os.environ, **cluster_env} + env_vars = {**os.environ, **cluster_env} + # Per-run sandbox flags travel to workers via env, not cfg — a + # run flag must never perturb the content-addressed job identity. + if no_sandbox: + env_vars[NO_SANDBOX_ENV] = "1" + if require_sandbox: + env_vars[REQUIRE_SANDBOX_ENV] = require_sandbox if verbose: console.print(f"[dim]$ {' '.join(cmd)}[/dim]") sys.exit( _run_snakemake( - cmd, env=env, scratch_root=rundirs.root, verbose=verbose + cmd, env=env_vars, scratch_root=rundirs.root, verbose=verbose ) ) @@ -966,23 +861,17 @@ def _build_snakemake_cmd( force: bool, has_outputs: bool, ) -> list[str]: - """Build the snakemake argv list for ``lc run``. + """Build the snakemake argv list for ``lc materialize``. ``--rerun-triggers`` uses ``nargs=+`` in snakemake's argparse, so without an explicit ``--`` separator it greedily consumes the first positional target path as an extra trigger value, causing an "invalid choice" error. ``--shared-fs-usage`` lists everything *except* - ``software-deployment``. With it included (snakemake's default), - spawned job commands embed the *driver's* ``sys.executable`` — a - path that doesn't exist inside a Dask Gateway worker image. Without - it, workers invoke plain ``python`` from their own environment, - which is equally correct on the other backends: LocalCluster - threads and srun-launched SLURM workers inherit the driver's - activated environment (and SLURM setups already require it — see - the ``dask``-on-PATH check in the cluster module). One invocation - shape for every backend; everything else stays shared via the - common filesystem (persistence, inputs/outputs, sources). + ``software-deployment``: with it included (snakemake's default), + spawned job commands embed the *driver's* ``sys.executable``; + without it, workers invoke plain ``python`` from their own + environment — one invocation shape everywhere. """ cmd: list[str] = [ "snakemake", @@ -1016,12 +905,15 @@ def _build_snakemake_cmd( return cmd -def _target_for(project: Path, output_id: str, universe: str) -> str: - """Translate an output id into a Snakemake target path (the manifest). +def _targets_for( + project: Path, output_ids: tuple[str, ...], universes: list[str] +) -> list[str]: + """Translate output ids into Snakemake target paths (the manifests). - Accepts either a bare ``output_id`` (root-level or unique sub-analysis - output) or a qualified ``analysis_id.output_id`` to disambiguate when - the same id appears in multiple sub-analyses. + Accepts bare ``output_id``s (root-level or unique sub-analysis + outputs) or qualified ``analysis_id.output_id`` to disambiguate when + the same id appears in multiple sub-analyses. The spec is parsed + once; matching is universe-independent, only the paths expand. """ from astra.helpers import load_yaml, resolve_analysis_tree @@ -1029,31 +921,141 @@ def _target_for(project: Path, output_id: str, universe: str) -> str: from lightcone.engine.tree import collect_tree_outputs, resolve_output_path spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - matches = [] - for to in collect_tree_outputs(spec): - if to.output_def.get("recipe") is None: - continue - qualified = ( - f"{to.analysis_id}.{to.output_id}" if to.analysis_id else to.output_id - ) - if qualified == output_id or to.output_id == output_id: - matches.append((qualified, to)) + tree_outputs = [ + to + for to in collect_tree_outputs(spec) + if to.output_def.get("recipe") is not None + ] - if not matches: + targets: list[str] = [] + for output_id in output_ids: + matches = [ + to + for to in tree_outputs + if output_id in (to.qualified_id, to.output_id) + ] + if not matches: + raise click.ClickException( + f"Output '{output_id}' not found in astra.yaml or has no recipe." + ) + if len(matches) > 1: + opts = ", ".join(to.qualified_id for to in matches) + raise click.ClickException( + f"Output '{output_id}' is ambiguous; qualify it as one of: {opts}" + ) + (to,) = matches + for universe in universes: + target = ( + resolve_output_path(project, to, universe) + / to.output_id + / MANIFEST_FILENAME + ) + targets.append(str(target.relative_to(project))) + return targets + + +# ============================================================================= +# lc run — the probe verb +# ============================================================================= + + +@main.command(context_settings={"ignore_unknown_options": True}) +@click.option( + "--no-sandbox", + is_flag=True, + help="Run the probe without the sandbox (recorded as unsandboxed).", +) +@click.option( + "--sandbox-debug", + is_flag=True, + help="Open a shell inside the sandbox to diagnose denials.", +) +@click.argument("cmd", nargs=-1, type=click.UNPROCESSED) +def run(no_sandbox: bool, sandbox_debug: bool, cmd: tuple[str, ...]) -> None: + """Run CMD inside the recipe environment (a probe). + + The command executes with the project's locked environment — the + same interpreter and packages recipes see — via + ``uv run --locked --exact``. Probes never materialize outputs; use + ``lc materialize`` for that. With no CMD, opens a shell in the + recipe environment. + """ + from astra.helpers import load_yaml, resolve_analysis_tree + + from lightcone.engine.tree import collect_tree_outputs + + project = _project_root() + spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) + tree_outputs = collect_tree_outputs(spec) + + # Rename guard (v6 reassigned `lc run` from pipeline execution to + # probing): a first argument naming a declared output errors before + # any exec — silently exec'ing `best_fit` as a command would be a + # far worse failure mode than a pointed redirect. + declared_ids = {to.output_id for to in tree_outputs} | { + to.qualified_id for to in tree_outputs + } + if cmd and not cmd[0].startswith("-") and cmd[0] in declared_ids: raise click.ClickException( - f"Output '{output_id}' not found in astra.yaml or has no recipe." + "outputs are materialized, not run — did you mean: " + f"`lc materialize {cmd[0]}`?" ) - if len(matches) > 1: - opts = ", ".join(q for q, _ in matches) - raise click.ClickException( - f"Output '{output_id}' is ambiguous; qualify it as one of: {opts}" + + env = _load_env(project) + _assert_inside_image_for_containerized(env.mode) + + if not cmd: + note = " (sandboxed)" if not no_sandbox else "" + console.print( + f"[dim]opening a shell inside the recipe environment{note}[/dim]" ) + cmd = (os.environ.get("SHELL") or "bash",) - _, to = matches[0] - target = ( - resolve_output_path(project, to, universe) / to.output_id / MANIFEST_FILENAME + if no_sandbox: + proc = subprocess.run( + [ + "uv", "run", "--locked", "--exact", + "--project", str(project), "--", *cmd, + ], + cwd=project, + ) + sys.exit(proc.returncode) + + # Sandboxed probe: byte-for-byte the recipe boundary — read scope is + # the project plus the union of declared external inputs, write + # scope is the tmp scope only (never in-tree). uv stays trusted + # plumbing outside the boundary; the shim restricts, then execs CMD. + from lightcone.engine.boundary import ExecScope + from lightcone.engine.contract import recipe_env_prefix + from lightcone.engine.image.mounts import external_input_paths + from lightcone.engine.sandbox.policy import build_policy + from lightcone.engine.sandbox.probe import probe as _capability_probe + from lightcone.engine.sandbox.wrap import run_wrapped, wrap_argv + + if sandbox_debug: + console.print("[dim]opening a shell inside the sandbox[/dim]") + cmd = (os.environ.get("SHELL") or "bash",) + + scope = ExecScope( + project_root=project, + output_dir=None, + read_paths=external_input_paths(project, spec), + ) + capability = _capability_probe() + policy = build_policy(scope, env_prefix=recipe_env_prefix(project)) + wrapped = wrap_argv( + tuple(cmd), + policy, + capability, + interpreter=( + "uv", "run", "--locked", "--exact", + "--project", str(project), "--", "python", + ), ) - return str(target.relative_to(project)) + result = run_wrapped( + wrapped, policy, cwd=project, env=dict(os.environ), capture=False + ) + sys.exit(result.returncode) # ============================================================================= @@ -1071,14 +1073,26 @@ def _target_for(project: Path, output_id: str, universe: str) -> str: ) def status(universe: str | None, as_json: bool) -> None: """Report materialization status for every declared output.""" + from astra.helpers import load_yaml, resolve_analysis_tree + from lightcone.engine.snakefile import discover_universes from lightcone.engine.status import get_output_status project = _project_root() + env = _load_env(project) universes = [universe] if universe else discover_universes(project) + spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) + + # One walk serves display, JSON, and the blast-radius count alike. + statuses = { + u: list(get_output_status(project, universe_id=u, env=env, spec=spec)) + for u in universes + } if as_json: payload = { + "mode": str(env.mode), + "env_version": env.env_version, "universes": [ { "universe_id": u, @@ -1089,7 +1103,7 @@ def status(universe: str | None, as_json: bool) -> None: "status": s.status, "recipe_command": s.recipe_command, } - for s in get_output_status(project, universe_id=u) + for s in statuses[u] ], } for u in universes @@ -1098,19 +1112,65 @@ def status(universe: str | None, as_json: bool) -> None: click.echo(json.dumps(payload, indent=2)) return + _print_status_header(env) + for u in universes: console.print(f"\n[bold]Universe[/bold] [cyan]{u}[/cyan]") - for s in get_output_status(project, universe_id=u): + for s in statuses[u]: label = _status_label(s.status) scope = f"[dim]{s.analysis_id}.[/dim]" if s.analysis_id else "" console.print(f" {label} {scope}{s.output_id}") + n_stale = sum( + 1 + for per_universe in statuses.values() + for s in per_universe + if s.manifest is not None + and s.manifest.get("env_version") not in (None, env.env_version) + ) + if n_stale > 0: + console.print( + f"\n[yellow]environment changed:[/yellow] {n_stale} materialized " + "output(s) are now stale" + ) + + +def _print_status_header(env) -> None: # type: ignore[no-untyped-def] + """The three header lines: mode / image / sandbox. + + Offline and local-only by invariant — reads pyproject + the local + image record, never the network. + """ + from lightcone.engine.boundary import get_boundary + from lightcone.engine.environment import Mode + + if env.mode is Mode.CONTAINERIZED: + from lightcone.engine.image import image_status + + n = len(env.image.system_packages) if env.image else 0 + mode_line = f"containerized ({n} system package{'s' if n != 1 else ''})" + try: + info = image_status(env.root, env) + if info.built: + image_line = f"{info.tag} — built [dim]({info.image_id})[/dim]" + else: + image_line = f"{info.tag} — needs build (run `lc build`)" + except ImageError as e: + image_line = f"[yellow]{e}[/yellow]" + else: + mode_line = "direct" + image_line = "—" + console.print(f"[dim]mode:[/dim] {mode_line}") + console.print(f"[dim]image:[/dim] {image_line}") + console.print(f"[dim]sandbox:[/dim] {get_boundary().describe_host()}") + _STATUS_STYLES = { "ok": "[green]✓ ok[/green] ", "stale": "[yellow]✸ stale[/yellow] ", "missing": "[red]✗ miss[/red] ", "alias": "[dim]→ alias[/dim] ", + "pre_migration": "[magenta]⧗ pre-v2[/magenta]", } @@ -1127,22 +1187,27 @@ def _status_label(s: str) -> str: @click.option("--universe", "-u", default=None) def verify(universe: str | None) -> None: """Validate the provenance chain by recomputing hashes.""" + from astra.helpers import load_yaml, resolve_analysis_tree + from lightcone.engine.snakefile import discover_universes from lightcone.engine.verify import verify_outputs project = _project_root() universes = [universe] if universe else discover_universes(project) + spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) failed = 0 for u in universes: console.print(f"\n[bold]Universe[/bold] [cyan]{u}[/cyan]") - for r in verify_outputs(project, universe_id=u): + for r in verify_outputs(project, universe_id=u, spec=spec): + notes = f" [dim]({', '.join(r.notes)})[/dim]" if r.notes else "" if r.passed: - console.print(f" [green]✓ ok[/green] {r.output_id}") + console.print(f" [green]✓ ok[/green] {r.output_id}{notes}") else: failed += 1 console.print( - f" [red]✗ {r.failure}[/red] {r.output_id} [dim]{r.detail}[/dim]" + f" [red]✗ {r.failure}[/red] {r.output_id} " + f"[dim]{r.detail}[/dim]{notes}" ) if failed: @@ -1157,166 +1222,40 @@ def verify(universe: str | None) -> None: @main.command() -@click.option("--force", is_flag=True, help="Rebuild all images even if cached") -@click.option( - "--runtime", - default=None, - help=( - "docker | podman | podman-hpc | kubernetes " - "(overrides ~/.lightcone/config.yaml)" - ), -) -def build(force: bool, runtime: str | None) -> None: - """Build container images declared in astra.yaml. - - Containerfile syntax is Dockerfile syntax — we use ``docker``, - ``podman``, or ``podman-hpc`` directly. Each Containerfile builds to - an OCI image tagged ``lc--`` in the runtime's local - image store. Pre-built registry images (``python:3.12-slim``, - ``ghcr.io/foo/bar:tag``) are skipped — the runtime pulls them at - ``lc run`` time. - - On a deployment without a local OCI runtime (the ``kubernetes`` - runtime on a lightcone JupyterHub), the same command builds through - the deployment's GCP Cloud Build service instead and pushes - ``/lc-:`` — same content-addressed - identity, zero configuration. +@click.option("--force", is_flag=True, help="Rebuild the image even if cached") +def build(force: bool) -> None: + """Build the project's environment image (containerized mode). + + The image is generated from the locked environment — never + user-authored: pyproject.toml + uv.lock + [tool.lightcone.image] + render to a Containerfile, built with podman under a + content-addressed tag. Direct-mode projects have no image. """ - from lightcone.engine.container import load_runtime + from lightcone.engine.environment import Mode + from lightcone.engine.image import ensure_image project = _project_root() - resolved_runtime = runtime or load_runtime(project_path=project).runtime + env = _load_env(project) - if resolved_runtime == "none": + if env.mode is Mode.DIRECT: console.print( - "[yellow]No container runtime available " - "(checked docker, podman, podman-hpc). " - "Install one to build images, or set [cyan]container.runtime[/cyan] " - "in [cyan]~/.lightcone/config.yaml[/cyan].[/yellow]" + "direct mode — no image to build; declare " + r"[cyan]\[tool.lightcone.image][/cyan] in pyproject.toml to " + "containerize." ) return - _ensure_images(project, runtime=resolved_runtime, force=force) - console.print("[green]Done.[/green]") - - -def _ensure_images(project: Path, *, runtime: str, force: bool = False) -> list[str]: - """Build/pull every container image referenced in astra.yaml. - - Returns the distinct resolved images, in declaration order (local - tags or registry refs for Containerfile specs, prebuilt specs - as-is). No-op (and empty) when *runtime* is ``"none"``. - - Idempotent: skips images already present (local image store, or the - deployment registry on the ``kubernetes`` runtime — where builds go - through Cloud Build instead of a local OCI CLI). Used by ``lc build`` - (with ``--force`` exposed) and as a pre-flight by ``lc run`` so the - first invocation after editing a Containerfile doesn't fail mid-DAG - with a missing image. - """ - if runtime == "none": - return [] - - from astra.helpers import load_yaml, resolve_analysis_tree - - from lightcone.engine.container import ( - KUBERNETES, - build_image, - compute_image_tag, - image_exists_locally, - is_containerfile, - pull_image, + record = ensure_image( + project, + env, + force=force, + on_progress=lambda msg: console.print(f"[cyan]{msg}[/cyan]"), ) - from lightcone.engine.tree import collect_tree_outputs - - spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - project_name = (spec.get("name") or project.name).lower().replace(" ", "-") - - images: list[str] = [] - seen: set[str] = set() - for to in collect_tree_outputs(spec): - recipe = to.output_def.get("recipe") or {} - spec_str = ( - recipe.get("container") - or to.analysis_spec.get("container") - or spec.get("container") - ) - if not spec_str or spec_str in seen: - continue - seen.add(spec_str) - if not is_containerfile(spec_str, project): - images.append(spec_str) - if runtime == KUBERNETES: - # Nothing to materialize: worker pods pull registry - # images straight from their source. - continue - # Pull so ``lc run`` can use ``--pull=never`` without - # depending on the runtime's registry resolution. - if image_exists_locally(spec_str, runtime=runtime) and not force: - continue - console.print(f"[cyan]Pulling[/cyan] {spec_str} [dim](via {runtime})[/dim]") - pull_image(spec_str, runtime=runtime) - continue - - if runtime == KUBERNETES: - images.append(_cloudbuild_image(project, spec_str, project_name, force)) - continue - - containerfile = project / spec_str - tag = compute_image_tag(project_name, containerfile, project) - images.append(tag) - if image_exists_locally(tag, runtime=runtime) and not force: - continue - console.print( - f"[cyan]Building[/cyan] {spec_str} → {tag} [dim](via {runtime})[/dim]" - ) - build_image(tag, containerfile, project, runtime=runtime) - return images - - -def _cloudbuild_image( - project: Path, spec_str: str, project_name: str, force: bool -) -> str: - """Ensure one Containerfile's image via GCP Cloud Build; return its ref.""" - from lightcone.engine.cloudbuild import ( - CloudBuildError, - cloudbuild_available, - ensure_image, + console.print( + f"[green]✓[/green] {record.tag} — built " + f"[dim](image id {record.image_id[:19]}…, {record.platform})[/dim]" ) - if not cloudbuild_available(): - raise click.ClickException( - "No image build backend on this host: the kubernetes runtime " - "has no local OCI CLI and this environment does not provide " - "the Cloud Build contract (LIGHTCONE_REGISTRY + " - "LIGHTCONE_BUILD_BUCKET). On a lightcone JupyterHub these are " - "injected into every user pod — ask the hub admin." - ) - - status = console.status(f"[cyan]Ensuring image for[/cyan] {spec_str} …") - status.start() - - def on_progress(phase: str, detail: str) -> None: - note = f" [dim]{detail}[/dim]" if detail else "" - status.update( - f"[cyan]Ensuring image for[/cyan] {spec_str}: {phase}{note}" - ) - - try: - ref = ensure_image( - project, - spec_str, - project_name=project_name, - force=force, - on_progress=on_progress, - ) - except CloudBuildError as e: - raise click.ClickException(str(e)) - finally: - status.stop() - console.print(f"[green]✓[/green] Worker image: [cyan]{ref}[/cyan]") - return ref - # ============================================================================= # lc export @@ -1412,7 +1351,8 @@ def export_wrroc_cmd( console.print( "[yellow]Warning:[/yellow] no materialized outputs were found — " "the bundle contains only the workflow definition.\n" - " This usually means recipes haven't been run yet (try [cyan]lc run[/cyan]) " + " This usually means recipes haven't been run yet " + "(try [cyan]lc materialize[/cyan]) " "or the [cyan].lightcone-manifest.json[/cyan] sidecars are missing.\n" " Workflow-only bundles will not pass strict Provenance Run Crate " "validation; that profile requires at least one materialized run." diff --git a/src/lightcone/engine/__init__.py b/src/lightcone/engine/__init__.py index 27db6d3a..bef2c0cf 100644 --- a/src/lightcone/engine/__init__.py +++ b/src/lightcone/engine/__init__.py @@ -1,6 +1,12 @@ -"""Lightcone execution engine. +"""The lightcone execution engine.""" +from __future__ import annotations -Snakemake-backed orchestrator for materializing astra.yaml outputs. -Provenance is recorded in per-output content-addressed manifests -(``.lightcone-manifest.json``) co-located with each output. -""" + +def lc_version() -> str: + """The running lightcone-cli version ("unknown" for broken installs).""" + try: + from importlib.metadata import version + + return version("lightcone-cli") + except Exception: + return "unknown" diff --git a/src/lightcone/engine/attestation.py b/src/lightcone/engine/attestation.py new file mode 100644 index 00000000..b9822d95 --- /dev/null +++ b/src/lightcone/engine/attestation.py @@ -0,0 +1,93 @@ +"""Worker-side runtime attestation. + +Captures the environment-describing manifest fields (spec §3) from +*inside the process that runs the recipe* — which in containerized mode +is inside the image, so the scoping rule "environment-describing fields +are captured inside the boundary" holds by construction. + +Everything here is best-effort observation, never a gate: a field that +cannot be determined records ``None`` rather than failing the run. +""" +from __future__ import annotations + +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +#: Threading knobs that silently change numerical results' runtime +#: behaviour — recorded so a puzzled reader a year later can see them. +_THREAD_KNOBS = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", +) + + +def capture_runtime_attestation() -> dict[str, Any]: + """The attestation block merged into every manifest.""" + return { + "uv_version": _uv_version(), + "platform": { + "os_release": _os_release(), + "kernel": platform.release(), + "glibc": _glibc(), + "arch": platform.machine(), + }, + "python_build": _python_build(), + "env_snapshot": { + "locale": os.environ.get("LC_ALL") or os.environ.get("LANG"), + "tz": os.environ.get("TZ"), + **{k.lower(): os.environ.get(k) for k in _THREAD_KNOBS}, + }, + "gpu_driver": _gpu_driver(), + } + + +def _uv_version() -> str | None: + if not shutil.which("uv"): + return None + try: + out = subprocess.run( + ["uv", "--version"], capture_output=True, text=True, timeout=10 + ) + except (OSError, subprocess.TimeoutExpired): + return None + # "uv 0.12.3 (…)" → "0.12.3" + parts = out.stdout.split() + return parts[1] if out.returncode == 0 and len(parts) >= 2 else None + + +def _os_release() -> str | None: + try: + for line in Path("/etc/os-release").read_text().splitlines(): + if line.startswith("PRETTY_NAME="): + return line.partition("=")[2].strip().strip('"') + except OSError: + pass + if sys.platform == "darwin": + return f"macOS {platform.mac_ver()[0]}" + return None + + +def _glibc() -> str | None: + lib, version = platform.libc_ver() + return f"{lib} {version}" if lib else None + + +def _python_build() -> str: + impl = platform.python_implementation() + build = " ".join(platform.python_build()) + return f"{impl} {platform.python_version()} ({build})" + + +def _gpu_driver() -> str | None: + try: + text = Path("/proc/driver/nvidia/version").read_text() + return text.splitlines()[0].strip() if text else None + except OSError: + return None diff --git a/src/lightcone/engine/boundary.py b/src/lightcone/engine/boundary.py new file mode 100644 index 00000000..eb03fb9f --- /dev/null +++ b/src/lightcone/engine/boundary.py @@ -0,0 +1,86 @@ +"""The exec boundary — the seam where recipes meet enforcement. + +``run_rule`` never runs a recipe directly: it hands the command and its +declared scope to an :class:`ExecBoundary`. The sandbox layer provides +the implementation (Landlock on Linux, Seatbelt on macOS, in-container +Landlock under podman); a scope with ``sandbox: "off"`` runs bare and +attests honestly to ``mechanism: none`` — a manifest must never claim +enforcement that did not happen. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal, Protocol + + +@dataclass(frozen=True) +class ExecScope: + """What the recipe is declared to touch.""" + + project_root: Path + output_dir: Path | None # None for probes (no in-tree write scope) + read_paths: tuple[Path, ...] # declared inputs + writable_project: bool = False + sandbox: Literal["on", "off"] = "on" + + +@dataclass(frozen=True) +class SandboxAttestation: + """The hermeticity record for one exec — the applied flags, never + the documentation.""" + + mechanism: str # landlock|seatbelt|podman|podman+landlock|none + fs: str # declared|project-rw|os-only|open + network: str # denied|allowed|unenforced + landlock_abi: int | None = None + exec_allowlist_version: int | None = None + + def to_manifest(self) -> dict[str, Any]: + record: dict[str, Any] = { + "mechanism": self.mechanism, + "fs": self.fs, + "network": self.network, + } + if self.landlock_abi is not None: + record["landlock_abi"] = self.landlock_abi + if self.exec_allowlist_version is not None: + record["exec_allowlist_version"] = self.exec_allowlist_version + return record + + +@dataclass(frozen=True) +class BoundaryResult: + returncode: int + stdout: str + stderr: str + attestation: SandboxAttestation + #: Extra console lines the boundary wants surfaced (downgrade + #: notices, denial explanations) — the caller emits them verbatim. + notes: tuple[str, ...] = field(default=()) + + +class ExecBoundary(Protocol): + def probe(self, scope: ExecScope) -> SandboxAttestation: + """What enforcement WOULD apply to this scope on this host — + checked worker-side per job (the driver's kernel is not the + worker's), and how ``--require-sandbox`` refuses before exec.""" + ... + + def execute( + self, + command: str, + scope: ExecScope, + env: dict[str, str], + ) -> BoundaryResult: ... + + def describe_host(self) -> str: + """One line for the ``lc status`` sandbox header.""" + ... + + +def get_boundary() -> ExecBoundary: + """The active exec boundary (the sandbox layer's implementation).""" + from lightcone.engine.sandbox import SandboxExecBoundary + + return SandboxExecBoundary() diff --git a/src/lightcone/engine/cloudbuild.py b/src/lightcone/engine/cloudbuild.py deleted file mode 100644 index 02fdd0ee..00000000 --- a/src/lightcone/engine/cloudbuild.py +++ /dev/null @@ -1,420 +0,0 @@ -"""Remote image builds through GCP Cloud Build. - -The build backend for deployments where no OCI runtime exists on the -host — a JupyterHub user pod on GKE. ``lc build`` tars the project's -**staged build context** (the exact file set the content-addressed tag -hashes), uploads it to a deployment-provided GCS bucket, and submits a -Cloud Build job that pushes the image to the deployment's Artifact -Registry. Auth is the pod's Workload Identity, spoken to the GCE -metadata server — no stored credentials, no SDK dependency, no git -remote required. - -Image identity is the same content-addressed scheme as everywhere else -(:func:`lightcone.engine.container.image_identity`); the pushed ref is -``$LIGHTCONE_REGISTRY/lc-:`` and "is the image up to -date" is a single registry HEAD on that ref — unchanged files never -rebuild, never even upload. - -Deployment contract (env vars injected into user pods — see the -hub-deploy ``lightcone`` hub config): - -- :data:`~lightcone.engine.container.REGISTRY_ENV` — Artifact Registry - prefix (``-docker.pkg.dev//``); also names the - GCP project builds run in. -- :data:`BUCKET_ENV` — GCS bucket for build sources and logs. Its - presence (with the registry) is what selects this backend. -- :data:`SERVICE_ACCOUNT_ENV` (optional) — dedicated build service - account; the deployment grants it registry-writer rights only. - -The pod's identity needs ``cloudbuild.builds.editor``, -``iam.serviceAccountUser`` on the build SA, object create/view on the -bucket, and ``artifactregistry.reader`` for the freshness probe. -""" - -from __future__ import annotations - -import io -import json -import os -import tarfile -import tempfile -import time -import urllib.error -import urllib.parse -import urllib.request -from collections.abc import Callable -from pathlib import Path - -from lightcone.engine.container import ( - ContainerBuildError, - _populate_build_context, - deployment_registry, - image_identity, - registry_image_ref, -) - -#: GCS bucket for build sources/logs. Presence selects this backend. -BUCKET_ENV = "LIGHTCONE_BUILD_BUCKET" - -#: Optional dedicated Cloud Build service account (bare email or full -#: ``projects/…/serviceAccounts/…`` resource name). -SERVICE_ACCOUNT_ENV = "LIGHTCONE_BUILD_SERVICE_ACCOUNT" - -#: Hard ceiling on one build, seconds (also sent as the Cloud Build -#: timeout). Project images are slim; single-digit minutes is typical. -_BUILD_DEADLINE_S = 1800 - -_POLL_INTERVAL_S = 5.0 - -_METADATA_TOKEN_URL = ( - "http://metadata.google.internal/computeMetadata/v1/" - "instance/service-accounts/default/token" -) - -#: Progress callback ``(phase, detail)``; phases are ``cached``, -#: ``staging``, then Cloud Build statuses lowercased (queued/working/…). -ProgressFn = Callable[[str, str], None] - - -class CloudBuildError(ContainerBuildError): - """An image could not be produced through Cloud Build. - - Subclasses :class:`ContainerBuildError` so one handler covers every - way an image can fail to materialize, local or cloud. - """ - - -def cloudbuild_available() -> bool: - """Is this environment configured for Cloud Build image builds?""" - return bool(os.environ.get(BUCKET_ENV)) and deployment_registry() is not None - - -# --------------------------------------------------------------------------- -# Auth + HTTP plumbing -# --------------------------------------------------------------------------- - - -def _metadata_access_token() -> str | None: - """OAuth2 access token from the GCE metadata server, or ``None``. - - On GKE with Workload Identity this returns a token for the - Kubernetes service account's bound GCP identity. Off-GCP the - metadata host doesn't resolve and we return ``None`` quickly. - """ - req = urllib.request.Request( - _METADATA_TOKEN_URL, headers={"Metadata-Flavor": "Google"} - ) - try: - with urllib.request.urlopen(req, timeout=5) as resp: - payload = json.loads(resp.read().decode("utf-8")) - except (urllib.error.URLError, OSError, ValueError): - return None - token = payload.get("access_token") - return token if isinstance(token, str) and token else None - - -def _token() -> str: - token = _metadata_access_token() - if token is None: - raise CloudBuildError( - "No GCP credentials available from the metadata server. The " - "Cloud Build backend needs Workload Identity (or another " - "metadata-served identity) with cloudbuild.builds.editor." - ) - return token - - -def _request( - method: str, - url: str, - token: str, - *, - body: bytes | None = None, - content_type: str = "application/json", -) -> tuple[int, bytes]: - req = urllib.request.Request( - url, - data=body, - method=method, - headers={ - "Authorization": f"Bearer {token}", - **({"Content-Type": content_type} if body is not None else {}), - }, - ) - try: - with urllib.request.urlopen(req, timeout=120) as resp: - return resp.status, resp.read() - except urllib.error.HTTPError as exc: - return exc.code, exc.read() - except (urllib.error.URLError, OSError) as exc: - raise CloudBuildError(f"Could not reach {url.split('?')[0]} ({exc}).") from exc - - -def _json_or_error(status: int, payload: bytes, what: str) -> dict[str, object]: - if not 200 <= status < 300: - detail = payload.decode("utf-8", errors="replace")[:500] - raise CloudBuildError(f"{what} failed: HTTP {status}\n{detail}") - try: - parsed = json.loads(payload.decode("utf-8") or "{}") - except ValueError as exc: - raise CloudBuildError(f"{what} returned unparseable JSON.") from exc - return parsed if isinstance(parsed, dict) else {} - - -# --------------------------------------------------------------------------- -# Registry freshness probe -# --------------------------------------------------------------------------- - - -def registry_image_exists(ref: str) -> bool | None: - """Does *ref* exist in its registry? ``None`` when unknowable. - - Speaks the Docker Registry v2 API with the metadata-server token - (Artifact Registry accepts OAuth2 access tokens as Bearer). Returns - ``None`` — not ``False`` — when there are no credentials or the - registry can't be reached, so callers can distinguish "absent, - build it" from "can't tell". - """ - host, _, path = ref.partition("/") - repo, _, tag = path.rpartition(":") - if not (host and repo and tag): - return None - token = _metadata_access_token() - if token is None: - return None - url = f"https://{host}/v2/{repo}/manifests/{urllib.parse.quote(tag, safe='')}" - req = urllib.request.Request( - url, - method="HEAD", - headers={ - "Authorization": f"Bearer {token}", - "Accept": ( - "application/vnd.oci.image.index.v1+json, " - "application/vnd.oci.image.manifest.v1+json, " - "application/vnd.docker.distribution.manifest.v2+json, " - "application/vnd.docker.distribution.manifest.list.v2+json" - ), - }, - ) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return bool(200 <= resp.status < 300) - except urllib.error.HTTPError as exc: - return None if exc.code in (401, 403) else False - except (urllib.error.URLError, OSError): - return None - - -# --------------------------------------------------------------------------- -# Source staging + upload -# --------------------------------------------------------------------------- - - -def _staged_context_tarball(project: Path, containerfile: Path) -> bytes: - """gzip tarball of the staged build context (the hashed file set).""" - with tempfile.TemporaryDirectory(prefix="lc-cloudbuild-") as tmp: - staged = Path(tmp) - _populate_build_context(staged, containerfile, project) - buf = io.BytesIO() - with tarfile.open(fileobj=buf, mode="w:gz") as tar: - for entry in sorted(staged.rglob("*")): - tar.add(entry, arcname=str(entry.relative_to(staged))) - return buf.getvalue() - - -def _upload_source(bucket: str, object_name: str, data: bytes, token: str) -> None: - url = ( - "https://storage.googleapis.com/upload/storage/v1/b/" - f"{urllib.parse.quote(bucket, safe='')}/o?uploadType=media&name=" - f"{urllib.parse.quote(object_name, safe='')}" - ) - status, payload = _request( - "POST", url, token, body=data, content_type="application/gzip" - ) - _json_or_error(status, payload, "Source upload to the build bucket") - - -def _fetch_log_tail(bucket: str, build_id: str, token: str, lines: int = 30) -> str: - object_name = urllib.parse.quote(f"logs/log-{build_id}.txt", safe="") - url = ( - "https://storage.googleapis.com/storage/v1/b/" - f"{urllib.parse.quote(bucket, safe='')}/o/{object_name}?alt=media" - ) - try: - status, payload = _request("GET", url, token) - except CloudBuildError: - return "" - if not 200 <= status < 300: - return "" - text = payload.decode("utf-8", errors="replace") - return "\n".join(text.splitlines()[-lines:]) - - -# --------------------------------------------------------------------------- -# Build submission + polling -# --------------------------------------------------------------------------- - - -def _gcp_project(registry: str) -> str: - """GCP project id out of an Artifact Registry prefix. - - ``-docker.pkg.dev//`` → ````. - """ - parts = registry.split("/") - if len(parts) < 2 or not parts[0].endswith("-docker.pkg.dev"): - raise CloudBuildError( - f"{registry!r} is not an Artifact Registry prefix " - "(expected -docker.pkg.dev//); the " - "Cloud Build backend only targets Artifact Registry." - ) - return parts[1] - - -def _submit_build( - *, - gcp_project: str, - bucket: str, - source_object: str, - containerfile_name: str, - image_ref: str, - token: str, -) -> str: - """Create the build; return its id.""" - build: dict[str, object] = { - "source": {"storageSource": {"bucket": bucket, "object": source_object}}, - "steps": [ - { - "name": "gcr.io/cloud-builders/docker", - "args": ["build", "-t", image_ref, "-f", containerfile_name, "."], - } - ], - "images": [image_ref], - "timeout": f"{_BUILD_DEADLINE_S}s", - # Logs into our own bucket: GCS_ONLY is required when running as - # a custom service account, and it is where the failure tail - # comes from. - "logsBucket": f"gs://{bucket}/logs", - "options": {"logging": "GCS_ONLY"}, - } - build_sa = (os.environ.get(SERVICE_ACCOUNT_ENV) or "").strip() - if build_sa: - if "/" not in build_sa: - build_sa = f"projects/{gcp_project}/serviceAccounts/{build_sa}" - build["serviceAccount"] = build_sa - - url = f"https://cloudbuild.googleapis.com/v1/projects/{gcp_project}/builds" - status, payload = _request("POST", url, token, body=json.dumps(build).encode()) - op = _json_or_error(status, payload, "Cloud Build submission") - meta = op.get("metadata") - build_info = meta.get("build") if isinstance(meta, dict) else None - build_id = build_info.get("id") if isinstance(build_info, dict) else None - if not isinstance(build_id, str) or not build_id: - raise CloudBuildError( - f"Cloud Build submission returned no build id (response keys: {sorted(op)})." - ) - return build_id - - -def _wait_for_build( - gcp_project: str, - build_id: str, - token: str, - on_progress: ProgressFn | None, -) -> str: - """Poll until a terminal status; return it.""" - url = ( - f"https://cloudbuild.googleapis.com/v1/projects/{gcp_project}" - f"/builds/{build_id}" - ) - deadline = time.monotonic() + _BUILD_DEADLINE_S + 120 - last_status = "" - while time.monotonic() < deadline: - status_code, payload = _request("GET", url, token) - build = _json_or_error(status_code, payload, "Cloud Build status poll") - status = str(build.get("status") or "") - if status != last_status: - last_status = status - if on_progress: - on_progress(status.lower(), "") - if status in ( - "SUCCESS", - "FAILURE", - "INTERNAL_ERROR", - "TIMEOUT", - "CANCELLED", - "EXPIRED", - ): - return status - time.sleep(_POLL_INTERVAL_S) - raise CloudBuildError(f"Timed out waiting for Cloud Build {build_id} to finish.") - - -# --------------------------------------------------------------------------- -# High-level entry point -# --------------------------------------------------------------------------- - - -def ensure_image( - project: Path, - containerfile_spec: str, - *, - project_name: str, - force: bool = False, - on_progress: ProgressFn | None = None, -) -> str: - """Make sure the project's image is in the registry; return its ref. - - Content-addressed and git-free: the tag hashes the staged build - context, so an unchanged environment is a single registry HEAD (no - build, no upload), and any change builds from the working tree as - it is right now. *force* skips the freshness probe and rebuilds. - """ - containerfile = project / containerfile_spec - if not containerfile.is_file(): - raise CloudBuildError( - f"Declared container {containerfile_spec!r} not found in {project}." - ) - registry = deployment_registry() - bucket = (os.environ.get(BUCKET_ENV) or "").strip().removeprefix("gs://").rstrip("/") - if registry is None or not bucket: - raise CloudBuildError( - "This environment is not configured for Cloud Build: both " - f"LIGHTCONE_REGISTRY and {BUCKET_ENV} must be set (they are " - "injected by the deployment)." - ) - ref = registry_image_ref(project_name, containerfile, project, registry=registry) - - if not force and registry_image_exists(ref) is True: - if on_progress: - on_progress("cached", f"{ref} already in the registry") - return ref - - token = _token() - gcp_project = _gcp_project(registry) - - if on_progress: - on_progress("staging", "uploading build context") - # Content-addressed object name: identical contexts collide into - # the same object, which is exactly right. - _, digest = image_identity(project_name, containerfile, project) - source_object = f"sources/lc-{project_name}-{digest}.tar.gz" - _upload_source( - bucket, source_object, _staged_context_tarball(project, containerfile), token - ) - - build_id = _submit_build( - gcp_project=gcp_project, - bucket=bucket, - source_object=source_object, - containerfile_name=containerfile.name, - image_ref=ref, - token=token, - ) - status = _wait_for_build(gcp_project, build_id, token, on_progress) - if status != "SUCCESS": - tail = _fetch_log_tail(bucket, build_id, token) - raise CloudBuildError( - f"Cloud Build {build_id} ended with status {status}." - + (f" Last build output:\n{tail}" if tail else "") - ) - return ref diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py deleted file mode 100644 index 890d6be5..00000000 --- a/src/lightcone/engine/container.py +++ /dev/null @@ -1,877 +0,0 @@ -"""Container runtime layer. - -We commit to **Dockerfile syntax** for ``Containerfile`` and **own** the -container invocation end-to-end — Snakemake's built-in ``container:`` -directive and ``--sdm apptainer`` pipeline are deliberately not used. A -single config knob picks the OCI runtime; building and running both go -through it. - -Two surfaces: - -* :func:`compute_image_tag` and :func:`build_image` cover the **build** - phase — ``lc build`` invokes them to produce ``lc--`` - in the runtime's local image store. - -* :func:`wrap_recipe` covers the **run** phase — the Snakefile generator - calls it to convert a raw recipe into a shell command that executes - inside the configured container runtime. - -Supported runtimes: - * ``docker`` / ``podman`` — local desktop or build host - * ``podman-hpc`` — NERSC-style login nodes; ``build`` migrates the - image so compute-node apptainer can read it. ``run`` still uses - ``podman-hpc`` directly. - * ``kubernetes`` — the execution environment (a Dask Gateway worker - pod) already *is* the container: ``lc run`` starts the cluster - with the project's image, so ``wrap_recipe`` is a passthrough and - images resolve to registry refs (``/lc-:``) - instead of local-store tags. Building goes through a remote - builder (:mod:`lightcone.engine.cloudbuild`), never a local OCI - CLI. - * ``none`` — no container; recipe runs on the host. Useful for - development and for projects that don't need isolation. -""" -from __future__ import annotations - -import hashlib -import json -import logging -import os -import re -import shlex -import shutil -import subprocess -import tempfile -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from pathlib import Path - -import yaml - -from lightcone.engine.site_registry import detect_current_site - -logger = logging.getLogger(__name__) - -#: Supported runtimes, in fallback detection order. The site registry can -#: move a site's declared ``container_runtime`` to the front (see -#: :func:`detect_runtime`). Order rationale: podman-hpc first because -#: anyone who installed the HPC wrapper did so on purpose and plain -#: podman would build images compute nodes can't read; then podman -#: (rootless, no daemon); docker last, gated behind a ``docker info`` -#: probe so a down daemon doesn't silently win over a healthy podman. -RUNTIMES: tuple[str, ...] = ("podman-hpc", "podman", "docker") - -#: The non-OCI-CLI runtime: recipes run directly inside a worker pod -#: that was started from the project's image. Never auto-detected from -#: PATH — it is selected by site detection (a Dask Gateway deployment) -#: or pinned explicitly in ``~/.lightcone/config.yaml``. -KUBERNETES = "kubernetes" - -#: Registry prefix images are pushed to / pulled from on a deployment -#: with a remote builder (e.g. ``europe-west1-docker.pkg.dev// -#: `` on a lightcone JupyterHub). Injected into user pods by the -#: deployment; its presence is half of the Cloud Build contract (see -#: :mod:`lightcone.engine.cloudbuild`). -REGISTRY_ENV = "LIGHTCONE_REGISTRY" - -#: Files whose contents contribute to the image tag hash. -DEPENDENCY_FILES = ( - "requirements.txt", - "requirements-dev.txt", - "requirements-test.txt", - "pyproject.toml", - "setup.py", - "setup.cfg", - "poetry.lock", - "Pipfile.lock", - "uv.lock", - "conda-lock.yml", - "environment.yml", - "environment.yaml", -) - -#: Matches a Dockerfile-style flag like ``--from=builder`` or ``--chown=u:g``. -_FLAG_RE = re.compile(r"^--[A-Za-z][A-Za-z0-9-]*(=\S+)?$") - - -class ContainerBuildError(Exception): - """Raised when a container image build fails.""" - - -@dataclass -class ContainerBuildResult: - """Result of building a container image.""" - - tag: str - already_existed: bool - exit_code: int = 0 - stdout: str = "" - stderr: str = "" - - -@dataclass -class ContainerStatus: - """Status information for a container spec.""" - - type: str # "none", "prebuilt", "build" - image: str | None = None - exists: bool | None = None - containerfile: str | None = None - - -@dataclass(frozen=True) -class RuntimeChoice: - """Result of resolving the container runtime to use. - - ``runtime`` is the resolved value (``docker | podman | podman-hpc | none``). - ``explicit`` is ``True`` when the user pinned this value in - ``~/.lightcone/config.yaml`` — i.e. they typed ``runtime: docker``, - ``runtime: podman``, … or ``runtime: none``. ``False`` means - ``runtime: auto`` (or no config), and the runtime is whatever - detection produced — including ``none`` as a silent fallback. - - Callers use ``explicit`` to decide whether silently running without - isolation is acceptable. When the user explicitly opted out, no - surprise. When auto fell back to ``none`` against the spec's - declared containers, the manifest's ``container_image`` field would - misrepresent what actually executed — that is a provenance hazard - and the caller should warn or refuse to proceed. - """ - - runtime: str - explicit: bool - - -# --------------------------------------------------------------------------- -# Runtime detection / config -# --------------------------------------------------------------------------- - - -def detect_runtime() -> str | None: - """Return the first usable runtime in :func:`_detection_order`, or ``None``. - - "Usable" means the binary is on PATH and (for docker) its daemon - answers ``docker info``. Site-declared preferences (e.g. Perlmutter - → podman-hpc) are *hints* — missing-from-PATH falls through to the - next candidate. Errors on missing-but-explicit user config are - :func:`load_runtime`'s job. - - A site that declares ``container_runtime: kubernetes`` (a Dask - Gateway deployment) short-circuits the PATH probing entirely — - there is no binary to find; the pod itself is the container. - """ - if _site_preferred_runtime() == KUBERNETES: - return KUBERNETES - for runtime in _detection_order(): - if shutil.which(runtime) is None: - continue - if runtime == "docker" and not _docker_daemon_up(): - continue - return runtime - return None - - -def _detection_order() -> tuple[str, ...]: - """RUNTIMES with the host site's preferred runtime moved to the front.""" - preferred = _site_preferred_runtime() - if preferred is None: - return RUNTIMES - return (preferred, *(r for r in RUNTIMES if r != preferred)) - - -def _site_preferred_runtime() -> str | None: - """Return the host site's declared ``container_runtime``, else ``None``. - - Returns ``None`` when no site matches, no preference is declared, or - the declared value is not a known runtime — never raises. - """ - preferred = detect_current_site().get("container_runtime") - return preferred if preferred in (*RUNTIMES, KUBERNETES) else None - - -def _docker_daemon_up() -> bool: - try: - result = subprocess.run( - ["docker", "info"], - capture_output=True, - timeout=5, - check=False, - ) - except (subprocess.TimeoutExpired, FileNotFoundError): - return False - return result.returncode == 0 - - -def _global_config_path() -> Path: - return Path.home() / ".lightcone" / "config.yaml" - - -def load_runtime(*, project_path: Path | None = None) -> RuntimeChoice: - """Resolve the container runtime to use. - - Reads ``container.runtime`` from ``~/.lightcone/config.yaml`` (the - project_path is accepted for future per-project overrides but is not - consulted today). Values: - - * ``auto`` (default) — first available runtime in :data:`RUNTIMES`, - else falls back to ``"none"`` with ``explicit=False``. On a site - that declares ``container_runtime: kubernetes``, auto resolves to - :data:`KUBERNETES` without any PATH probing. - * ``docker | podman | podman-hpc`` — explicit; binary must exist. - * ``kubernetes`` — explicit; no binary involved. - * ``none`` — explicit opt-out; recipes run on the host. - - Raises :class:`ContainerBuildError` if an explicit runtime is - configured but its binary is missing on PATH, or if the configured - value is unrecognised. - """ - cfg_path = _global_config_path() - requested = "auto" - if cfg_path.is_file(): - try: - data = yaml.safe_load(cfg_path.read_text()) or {} - requested = (data.get("container") or {}).get("runtime") or "auto" - except yaml.YAMLError: - logger.warning("Could not parse %s; using runtime: auto", cfg_path) - requested = "auto" - - if requested == "auto": - return RuntimeChoice(runtime=detect_runtime() or "none", explicit=False) - if requested in ("none", KUBERNETES): - return RuntimeChoice(runtime=requested, explicit=True) - if requested not in RUNTIMES: - raise ContainerBuildError( - f"Unknown container.runtime {requested!r} in {cfg_path}. " - f"Expected one of: auto, none, {KUBERNETES}, {', '.join(RUNTIMES)}." - ) - if shutil.which(requested) is None: - raise ContainerBuildError( - f"Configured container.runtime {requested!r} is not on PATH. " - f"Install {requested} or set container.runtime to a different value " - f"in {cfg_path}." - ) - return RuntimeChoice(runtime=requested, explicit=True) - - -# --------------------------------------------------------------------------- -# Image tag computation -# --------------------------------------------------------------------------- - - -def find_dependency_files(project_path: Path) -> list[Path]: - """Return sorted list of dependency files found in *project_path*.""" - found = [project_path / name for name in DEPENDENCY_FILES] - return sorted(p for p in found if p.is_file()) - - -def _hash_file_into(path: Path, h: hashlib._Hash) -> None: - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(64 * 1024), b""): - h.update(chunk) - - -def _hash_named_file(path: Path, label: str, h: hashlib._Hash) -> None: - """Mix *path*'s identity and contents into *h* with explicit framing. - - Path-prefix + null separators stop boundary-shifting collisions — - e.g. moving a line from ``requirements.txt`` to ``requirements-dev.txt`` - no longer yields the same digest as keeping it in place. - """ - h.update(label.encode("utf-8")) - h.update(b"\0") - h.update(path.name.encode("utf-8")) - h.update(b"\0") - _hash_file_into(path, h) - h.update(b"\0") - - -def hash_file_contents(files: list[Path]) -> str: - """Return a SHA-256 hex digest over the framed contents of *files*. - - The digest mixes each file's basename and a label byte in addition to - its contents, so reordering or relabelling produces different digests. - """ - h = hashlib.sha256() - for f in files: - _hash_named_file(f, "f", h) - return h.hexdigest() - - -def _iter_build_context_entries( - containerfile: Path, project_path: Path -) -> Iterator[tuple[str, Path]]: - """Yield ``(kind, path)`` for everything that contributes to a build. - - ``kind`` is one of ``"containerfile"``, ``"dep"``, ``"copy_file"``. - Sharing this iteration between :func:`compute_image_tag` and - :func:`_populate_build_context` guarantees by construction that the - hashed set and the staged set cover identical files — so the tag - can never invalidate against a stage that's missing inputs (or vice - versa). - - Sources behind ``--from=`` and URL/git ``ADD`` arguments are - skipped — they're not part of the host context. Directory sources - (including ``COPY . .``) are rejected: the image is an environment, - not a code snapshot — recipes run against the live project tree - (bind-mounted locally, shared filesystem on a hub), so baking - source directories in would only force pointless rebuilds and go - stale between them. - """ - text = containerfile.read_text(errors="replace") - copy_files: list[Path] = [] - bad: list[str] = [] - for src_str in _parse_copy_sources(text): - is_dir_source = False - for resolved in _expand_copy_source(src_str, project_path): - if resolved.is_dir(): - is_dir_source = True - elif resolved.is_file(): - copy_files.append(resolved) - if is_dir_source: - bad.append(src_str) - if bad: - raise ContainerBuildError( - f"{containerfile.name}: COPY/ADD of a directory " - f"({', '.join(repr(s) for s in bad)}) is not supported. The " - "image is a pure environment — recipes run against the live " - "project tree (bind-mounted locally, shared filesystem on a " - "hub), so project source never needs to be baked in. Remove " - "the line (e.g. `COPY . .`), or COPY individual files if the " - "build itself needs them." - ) - yield "containerfile", containerfile - for dep in find_dependency_files(project_path): - yield "dep", dep - for resolved in copy_files: - yield "copy_file", resolved - - -def directory_copy_sources(containerfile: Path, project_path: Path) -> list[str]: - """``COPY``/``ADD`` sources in *containerfile* that resolve to directories. - - Directory sources are unsupported (the image is an environment, not - a code snapshot); this is the shared detector behind the build-time - rejection in :func:`_iter_build_context_entries` and the advisory - warning in ``lc init``. - """ - text = containerfile.read_text(errors="replace") - bad: list[str] = [] - for src_str in _parse_copy_sources(text): - if any( - resolved.is_dir() - for resolved in _expand_copy_source(src_str, project_path) - ): - bad.append(src_str) - return bad - - -def image_identity( - project_name: str, - containerfile: Path, - project_path: Path, -) -> tuple[str, str]: - """Compute a content-addressed image identity ``(safe_name, digest)``. - - The digest (12-char sha256) covers the Containerfile, every - dependency file in :data:`DEPENDENCY_FILES`, and the contents of - every ``COPY``/``ADD`` source file referenced from the - Containerfile (directory sources are rejected — the image is an - environment, not a code snapshot). - - The identity is spelled two ways downstream — ``lc--`` - in a local image store (:func:`compute_image_tag`), ``…/lc-: - `` in a registry (:func:`registry_image_ref`) — but it is - one identity: the same digest everywhere, on every backend. - """ - h = hashlib.sha256() - for kind, path in _iter_build_context_entries(containerfile, project_path): - if kind == "containerfile": - _hash_named_file(path, "containerfile", h) - elif kind == "dep": - _hash_named_file(path, "dep", h) - else: # copy_file - rel = _safe_relpath(path, project_path) - h.update(b"copy\0") - h.update(rel.encode("utf-8")) - h.update(b"\0file\0") - _hash_file_into(path, h) - h.update(b"\0") - - return project_name.lower().replace(" ", "-"), h.hexdigest()[:12] - - -def compute_image_tag( - project_name: str, - containerfile: Path, - project_path: Path, -) -> str: - """Content-addressed local-store tag: ``lc--``. - - See :func:`image_identity` for what the digest covers. - """ - safe_name, digest = image_identity(project_name, containerfile, project_path) - return f"lc-{safe_name}-{digest}" - - -def registry_image_ref( - project_name: str, - containerfile: Path, - project_path: Path, - *, - registry: str, -) -> str: - """Content-addressed registry ref: ``/lc-:``. - - Same identity as :func:`compute_image_tag`, spelled for a registry: - the digest moves into the tag position so one repository per project - accumulates its image history. - """ - safe_name, digest = image_identity(project_name, containerfile, project_path) - return f"{registry.rstrip('/')}/lc-{safe_name}:{digest}" - - -def deployment_registry() -> str | None: - """The deployment-injected registry prefix, or ``None`` off-deployment.""" - registry = (os.environ.get(REGISTRY_ENV) or "").strip() - return registry.rstrip("/") or None - - -def runtime_registry(runtime: str) -> str | None: - """Registry prefix image identities resolve against under *runtime*. - - The single source of truth for "which spelling of the image identity - does this runtime use": the deployment registry on - :data:`KUBERNETES` (worker pods pull from a registry), ``None`` — - local-store tags — everywhere else. Shared by the Snakefile - generator and the status walker so their ``code_version``s can - never disagree about the image identity. - """ - return deployment_registry() if runtime == KUBERNETES else None - - -def _safe_relpath(path: Path, root: Path) -> str: - try: - return path.resolve().relative_to(root.resolve()).as_posix() - except ValueError: - return path.name - - -def _parse_copy_sources(containerfile_text: str) -> list[str]: - """Return raw source strings from ``COPY``/``ADD`` lines. - - Skips ``--from=`` copies (those reference another build stage, - not the host context) and URL/git arguments (network resources, not - part of the local context we can hash). Glob patterns and relative - paths are returned verbatim — :func:`_expand_copy_source` resolves - them against the project tree. - - Handles backslash line continuations and the JSON exec form - (``COPY ["src", "dest"]``). Heredoc COPY (``COPY < list[Path]: - """Resolve a ``COPY``/``ADD`` source to actual paths under *project_path*. - - Returns ``[project_path]`` for ``.`` (whole context). Globs are - expanded against *project_path*. Paths that escape the project root - are dropped — we don't hash arbitrary host filesystem. - """ - src = src.lstrip("/") - if not src or src == ".": - return [project_path] - if any(c in src for c in "*?["): - return sorted(project_path.glob(src)) - candidate = (project_path / src).resolve() - try: - candidate.relative_to(project_path.resolve()) - except ValueError: - return [] - if candidate.exists(): - return [candidate] - return [] - - -def is_containerfile(spec: str, project_path: Path) -> bool: - """Return ``True`` if *spec* refers to an existing file (Containerfile).""" - return (project_path / spec).is_file() - - -# --------------------------------------------------------------------------- -# Build -# --------------------------------------------------------------------------- - - -def image_exists_locally(tag: str, *, runtime: str) -> bool: - """Check whether *tag* exists in the runtime's local image store.""" - if runtime == "podman-hpc": - return image_exists_podman_hpc(tag) - try: - result = subprocess.run( - [runtime, "image", "inspect", tag], - capture_output=True, - check=False, - ) - return result.returncode == 0 - except FileNotFoundError: - return False - - -def image_exists_podman_hpc(tag: str) -> bool: - try: - result = subprocess.run( - ["podman-hpc", "image", "exists", tag], - capture_output=True, - check=False, - ) - return result.returncode == 0 - except FileNotFoundError: - return False - - -def _populate_build_context( - staged: Path, containerfile: Path, source_context: Path -) -> None: - """Mirror the Containerfile + its referenced sources into *staged*. - - Why this exists: NERSC's home and CFS filesystems are mounted via - Cray DVS, which doesn't implement ``llistxattr`` (returns ``EPROTO``). - Buildah's copier — used by ``podman``, ``podman-hpc``, and any other - buildah-backed runtime — calls ``llistxattr`` unconditionally on every - ``COPY`` source and crashes when the project lives on DVS. Staging - the build context into ``$TMPDIR`` (tmpfs on Linux) sidesteps the - issue entirely without forcing the user to relocate their project. - - The set of staged files is :func:`_iter_build_context_entries` — the - same iteration :func:`compute_image_tag` hashes — so the tag can't - invalidate against a stage that's missing files. - """ - src_root = source_context.resolve() - for kind, path in _iter_build_context_entries(containerfile, source_context): - if kind in ("containerfile", "dep"): - shutil.copy2(path, staged / path.name) - continue - try: # copy_file - rel = path.resolve().relative_to(src_root) - except ValueError: - continue - dest = staged / rel - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, dest) - - -def build_image( - tag: str, - containerfile: Path, - context: Path, - *, - runtime: str, - build_args: dict[str, str] | None = None, -) -> ContainerBuildResult: - """Build a container image with the given *runtime*. - - The build context is staged into a fresh tempdir before invocation - (see :func:`_populate_build_context`). For ``podman-hpc``, the image - is automatically migrated after build so compute nodes can access it. - - Raises :class:`ContainerBuildError` on failure. - """ - if runtime not in RUNTIMES: - raise ContainerBuildError( - f"Unsupported build runtime {runtime!r}; expected one of {RUNTIMES}." - ) - - with tempfile.TemporaryDirectory(prefix="lc-build-") as staged_str: - staged = Path(staged_str) - _populate_build_context(staged, containerfile, context) - staged_cf = staged / containerfile.name - cmd: list[str] = [runtime, "build", "-t", tag, "-f", str(staged_cf)] - for key, value in (build_args or {}).items(): - cmd += ["--build-arg", f"{key}={value}"] - cmd.append(str(staged)) - - try: - proc = subprocess.run(cmd, capture_output=True, text=True, check=False) - except FileNotFoundError: - raise ContainerBuildError( - f"{runtime} is not installed or not on PATH. " - f"Install {runtime} to build container images." - ) - - if proc.returncode != 0: - raise ContainerBuildError( - f"{runtime} build failed (exit code {proc.returncode}):\n{proc.stderr}" - ) - - if runtime == "podman-hpc": - _podman_hpc_migrate(tag) - - return ContainerBuildResult( - tag=tag, - already_existed=False, - exit_code=proc.returncode, - stdout=proc.stdout, - stderr=proc.stderr, - ) - - -def pull_image(image: str, *, runtime: str) -> None: - """Pull *image* into the runtime's local image store. - - Used by ``lc build`` so that pre-built registry images (e.g. - ``python:3.12-slim``) are present before ``lc run`` invokes the - runtime with ``--pull=never``. - - Raises :class:`ContainerBuildError` on failure or if *runtime* isn't - on PATH. - """ - if runtime not in RUNTIMES: - raise ContainerBuildError( - f"Unsupported runtime {runtime!r}; expected one of {RUNTIMES}." - ) - try: - proc = subprocess.run( - [runtime, "pull", image], - capture_output=True, - text=True, - check=False, - ) - except FileNotFoundError: - raise ContainerBuildError( - f"{runtime} is not installed or not on PATH." - ) - if proc.returncode != 0: - raise ContainerBuildError( - f"{runtime} pull {image} failed (exit code {proc.returncode}):\n" - f"{proc.stderr}" - ) - if runtime == "podman-hpc": - _podman_hpc_migrate(image) - - -def _podman_hpc_migrate(tag: str) -> None: - """Run ``podman-hpc migrate `` to make image available on compute nodes.""" - try: - proc = subprocess.run( - ["podman-hpc", "migrate", tag], - capture_output=True, - text=True, - check=False, - ) - except FileNotFoundError: - raise ContainerBuildError("podman-hpc not found — cannot migrate image.") - if proc.returncode != 0: - raise ContainerBuildError( - f"podman-hpc migrate failed (exit code {proc.returncode}):\n{proc.stderr}" - ) - logger.info("podman-hpc migrate %s succeeded.", tag) - - -# --------------------------------------------------------------------------- -# Run-time recipe wrap -# --------------------------------------------------------------------------- - - -def resolve_image_for_run( - spec: str | None, - *, - project_path: Path, - project_name: str, - registry: str | None = None, -) -> str | None: - """Translate an astra.yaml ``container:`` value into the image tag - that the runtime will execute. - - * ``None`` / empty → ``None`` (no container). - * Path to a Containerfile in the project → the content-addressed - identity ``lc build`` produces: a local-store tag - (``lc--``), or with *registry* set (a deployment with - a remote builder) the registry ref (``/lc-:``). - * Anything else (registry image, e.g. ``python:3.12-slim``, or a - pre-namespaced ``ghcr.io/foo/bar:tag``) → returned as-is for the - runtime to pull. - """ - if not spec: - return None - if is_containerfile(spec, project_path): - containerfile = project_path / spec - if registry is not None: - return registry_image_ref( - project_name, containerfile, project_path, registry=registry - ) - return compute_image_tag(project_name, containerfile, project_path) - return spec - - -def make_image_tag_resolver( - project_path: Path, - project_name: str, - *, - registry: str | None = None, -) -> Callable[[str | None], str | None]: - """Memoizing wrapper around :func:`resolve_image_for_run`. - - Multiple outputs typically share the same Containerfile, and resolving - a Containerfile path re-hashes it plus all dependency files - (lockfiles can be MB each). The returned closure caches by spec - string for the lifetime of the caller's loop. - """ - cache: dict[str | None, str | None] = {} - - def resolve(spec: str | None) -> str | None: - if spec in cache: - return cache[spec] - tag = resolve_image_for_run( - spec, - project_path=project_path, - project_name=project_name, - registry=registry, - ) - cache[spec] = tag - return tag - - return resolve - - -def wrap_recipe( - recipe: str, - *, - image: str | None, - runtime: str, -) -> str: - """Wrap *recipe* so it executes inside *image* under *runtime*. - - Returns a shell-command string suitable for Snakemake's ``shell()``. - Snakemake's ``{output[0]}`` / ``{input.X}`` / ``{wildcards.universe}`` - placeholders inside *recipe* are preserved — they substitute through - Python's ``str.format`` at execution time, after wrapping. - - No-op cases: - * *image* is ``None`` → recipe returned unchanged - * *runtime* is ``"none"`` → recipe returned unchanged - * *runtime* is :data:`KUBERNETES` → recipe returned unchanged: - the Dask worker pod executing it was started from *image*, so - wrapping would be containerizing twice. The image still flows - into ``code_version`` and the manifest — provenance records - the pod's image, which is what actually ran the recipe. - - The recipe is shell-quoted with :func:`shlex.quote` and passed as the - argument to ``bash -c`` inside the container, which keeps single - quotes, dollar signs, and other shell metacharacters intact across - the host bash → runtime CLI → container bash boundaries. - """ - if image is None or runtime in ("none", KUBERNETES): - return recipe - if runtime not in RUNTIMES: - raise ContainerBuildError( - f"Unsupported run runtime {runtime!r}; expected one of {RUNTIMES} or 'none'." - ) - inner = shlex.quote(recipe) - # ``--pull=never`` is critical for podman, which by default does - # short-name resolution against ``unqualified-search-registries`` - # in registries.conf — that fails for ``lc--`` tags - # produced by ``lc build`` even though the image sits in local - # storage. Telling the runtime not to fetch sidesteps the issue and - # is the same semantics on docker and podman-hpc. Registry images - # (``python:3.12-slim``, ``ghcr.io/...``) must be pulled in advance - # by ``lc build``. - # - # Bind-mount and chdir to $PWD so recipes that write to relative - # paths land in the project tree. Snakemake invokes us with - # cwd=project, so $PWD is the project root. - return ( - f'{runtime} run --rm --pull=never ' - f'-v "$PWD":"$PWD" -w "$PWD" ' - f'{image} bash -c {inner}' - ) - - -# --------------------------------------------------------------------------- -# Status -# --------------------------------------------------------------------------- - - -def get_container_status( - spec: str | None, - project_path: Path, - project_name: str, - *, - runtime: str, -) -> ContainerStatus: - """Return status information for a container spec without building.""" - if spec is None: - return ContainerStatus(type="none") - - if not is_containerfile(spec, project_path): - return ContainerStatus(type="prebuilt", image=spec) - - containerfile = project_path / spec - if runtime == KUBERNETES and (registry := deployment_registry()) is not None: - from lightcone.engine.cloudbuild import registry_image_exists - - ref = registry_image_ref( - project_name, containerfile, project_path, registry=registry - ) - return ContainerStatus( - type="build", - image=ref, - exists=registry_image_exists(ref), - containerfile=spec, - ) - tag = compute_image_tag(project_name, containerfile, project_path) - exists = ( - image_exists_locally(tag, runtime=runtime) - if runtime not in ("none", KUBERNETES) - else None - ) - return ContainerStatus( - type="build", - image=tag, - exists=exists, - containerfile=spec, - ) diff --git a/src/lightcone/engine/contract.py b/src/lightcone/engine/contract.py new file mode 100644 index 00000000..ce397688 --- /dev/null +++ b/src/lightcone/engine/contract.py @@ -0,0 +1,50 @@ +"""The launcher↔engine / driver↔worker environment-variable contract. + +One home for every ``LC_*`` variable that crosses a process boundary — +the launcher's delegation, the podman run wrapper, and the per-run +sandbox flags — so producer and consumers can never drift apart. (The +sandbox *shim*'s variables live in :mod:`lightcone._sandbox_exec`, +which must stay stdlib-only; the engine imports them from there.) +""" +from __future__ import annotations + +import os +from pathlib import Path + +#: Set on delegation (direct exec or podman re-entry) — the launcher +#: never delegates twice. Part of the frozen delegation interface. +DELEGATED_ENV = "LC_DELEGATED" + +#: "container" when the process runs inside the project image (set by +#: the podman run wrapper); how every layer answers "am I in the +#: image?". +WORKER_RUNTIME_ENV = "LC_WORKER_RUNTIME" +CONTAINER_RUNTIME_VALUE = "container" + +#: The network posture the container wrapper actually applied +#: ("none" | "host") — what the hermeticity record reports. +CONTAINER_NETWORK_ENV = "LC_CONTAINER_NETWORK" + +#: The driver-resolved image id, asserted by the worker's env check. +IMAGE_DIGEST_ENV = "LC_IMAGE_DIGEST" + +#: Per-run sandbox flags (env, not cfg: run flags must never perturb +#: the content-addressed job identity). +NO_SANDBOX_ENV = "LC_NO_SANDBOX" +REQUIRE_SANDBOX_ENV = "LC_REQUIRE_SANDBOX" + + +def in_container() -> bool: + return os.environ.get(WORKER_RUNTIME_ENV) == CONTAINER_RUNTIME_VALUE + + +def recipe_env_prefix(project_root: Path) -> Path: + """The recipe environment's prefix on this process's venue: the + baked ``/opt/venv`` inside the image, the project ``.venv`` + otherwise. The single home for that decision — sandbox policy and + probe assembly both use it.""" + if in_container(): + from lightcone.engine.image.constants import OPT_VENV + + return Path(OPT_VENV) + return project_root / ".venv" diff --git a/src/lightcone/engine/dask_cluster.py b/src/lightcone/engine/dask_cluster.py index cf6a69f0..ef6a92d1 100644 --- a/src/lightcone/engine/dask_cluster.py +++ b/src/lightcone/engine/dask_cluster.py @@ -1,42 +1,20 @@ # mypy: disable-error-code="no-untyped-call" -"""Cluster lifecycle for ``lc run``. +"""Cluster lifecycle for ``lc materialize``. -One context manager, four branches: +One run-scoped ``LocalCluster``, owned by the driver: the scheduler is +in-process, so its lifetime equals the run's lifetime — no service to +manage, no orphaned schedulers if the driver crashes. The child +snakemake's executor plugin reaches it through the +``DASK_SCHEDULER_ADDRESS`` env overlay yielded here. -- ``DASK_SCHEDULER_ADDRESS`` is already set → yield it as-is. We don't own - the cluster, so we don't tear it down. -- ``DASK_GATEWAY__ADDRESS`` is set (a JupyterHub/Dask Gateway - deployment) → **create** a run-scoped Gateway cluster with the - project's image and shut it down when the run finishes. Create/cull - per run is what makes image updates seamless: a Gateway cluster's - image is fixed at creation, so picking up a freshly built project - image *requires* a fresh cluster. Gateway scheduler addresses use a - custom ``gateway://`` comm scheme a bare ``distributed.Client`` - cannot dial, so this branch hands the executor the *cluster name* - (via :data:`GATEWAY_CLUSTER_ENV`) and the executor rejoins through - the authenticated Gateway API. -- ``SLURM_JOB_ID`` is set → start an in-process scheduler via - ``LocalCluster(n_workers=0)``, then ``srun`` one ``dask worker`` per node - across the allocation. Workers advertise the node's full resources; - per-rule ``threads`` / ``mem_mb`` / ``gpus`` map to per-task constraints. -- None of the above → ``LocalCluster()`` sized to the local machine. - -Outside the Gateway branch the scheduler is always in-process (driven -by ``lc run`` itself) so its lifetime equals the run's lifetime — no -service to manage, no orphaned schedulers if the driver crashes. On the -Gateway branch the Gateway server owns scheduling, and the same -lifetime contract is enforced there: the cluster is shut down on exit, -with the deployment's idle timeout as the backstop if lc dies uncleanly. +Additional venues (SLURM allocations, hub deployments) return later as +new branches behind this same context-manager seam. """ from __future__ import annotations -import getpass import logging import os -import shutil -import socket -import subprocess from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass @@ -48,22 +26,10 @@ RESOURCE_MEMORY = "memory" RESOURCE_GPUS = "gpus" -#: Parent→child rendezvous for the Gateway branch: ``cluster_for_run`` -#: sets this to the name of the cluster it created so the executor -#: plugin (running in the child snakemake process) can rejoin it via -#: ``Gateway().connect(name)``. Internal contract, not a user knob. -GATEWAY_CLUSTER_ENV = "LIGHTCONE_GATEWAY_CLUSTER" - -#: Bounds how long the Gateway branch waits for the first worker of a -#: cluster it created (seconds; default 600 — a first-time image pull -#: on a fresh node is minutes, not seconds). Without this bound an -#: unpullable image leaves the run sitting at zero workers forever. -GATEWAY_WORKER_TIMEOUT_ENV = "LIGHTCONE_GATEWAY_WORKER_TIMEOUT" - @dataclass class _NodeShape: - """Per-node resources advertised by the dask worker.""" + """Machine resources advertised by the dask worker.""" cpus: int mem_bytes: int @@ -71,31 +37,23 @@ class _NodeShape: def _detect_node_shape() -> _NodeShape: - """Read node capacity from SLURM env vars (with sensible fallbacks).""" - cpus = int(os.environ.get("SLURM_CPUS_ON_NODE") or os.cpu_count() or 1) - - mem_mb = os.environ.get("SLURM_MEM_PER_NODE") - if mem_mb: - mem_bytes = int(mem_mb) * 1_000_000 - else: - try: - import psutil # type: ignore[import-untyped] - - mem_bytes = psutil.virtual_memory().total - except ImportError: - mem_bytes = 0 # advisory: workers won't enforce memory caps + """Read machine capacity (with sensible fallbacks).""" + cpus = int(os.cpu_count() or 1) + try: + import psutil # type: ignore[import-untyped] - gpus = int(os.environ.get("SLURM_GPUS_ON_NODE") or 0) - return _NodeShape(cpus=cpus, mem_bytes=mem_bytes, gpus=gpus) + mem_bytes = psutil.virtual_memory().total + except ImportError: + mem_bytes = 0 # advisory: workers won't enforce memory caps + return _NodeShape(cpus=cpus, mem_bytes=mem_bytes, gpus=0) def _resource_dict(shape: _NodeShape) -> dict[str, float]: - """Resource keys advertised by a worker for this node shape. + """Resource keys advertised by the worker for this machine. - Single source of truth for which keys workers expose — both the - in-process LocalCluster and the srun-launched ``dask worker``s - advertise the same set so the executor's per-task requests resolve - on either path. + Workers must advertise every key the executor may request — Dask + matches by exact key presence — or rules with ``mem_mb`` / + ``gpus_per_task`` would never schedule. """ res: dict[str, float] = {RESOURCE_CPUS: float(shape.cpus)} if shape.mem_bytes: @@ -105,292 +63,25 @@ def _resource_dict(shape: _NodeShape) -> dict[str, float]: return res -def _resources_arg(shape: _NodeShape) -> str: - """Format `--resources` for `dask worker`.""" - return " ".join(f"{k}={int(v)}" for k, v in _resource_dict(shape).items()) - - -def gateway_branch_active() -> bool: - """Would :func:`cluster_for_run` take the Gateway branch right now? - - Exposed so ``lc run`` can shape the snakemake invocation (e.g. NFS - latency tolerance) before entering the cluster context. Pure - function of the environment, in the same priority order as the - branches in :func:`cluster_for_run`. - """ - if os.environ.get("DASK_SCHEDULER_ADDRESS"): - return False - return bool(os.environ.get("DASK_GATEWAY__ADDRESS")) - - @contextmanager def cluster_for_run( *, verbose: bool = False, local_directory: str | None = None, - worker_image: str | None = None, - max_workers: int | None = None, ) -> Iterator[dict[str, str]]: """Yield the env overlay the child snakemake needs to reach the cluster. - The parent (``lc run``) and the executor plugin live in different - processes, so connection info travels via environment variables. - Address-based branches yield ``{"DASK_SCHEDULER_ADDRESS": addr}``; - the Gateway branch yields ``{GATEWAY_CLUSTER_ENV: name}`` because - Gateway clusters are rejoined by name through the authenticated - Gateway API rather than dialled by address. + The parent (``lc materialize``) and the executor plugin live in + different processes, so connection info travels via the environment: + ``{"DASK_SCHEDULER_ADDRESS": addr}``. *local_directory*, when given, is where dask workers stage their - spilled task data and internal state files. ``lc run`` resolves it - to a path under :mod:`lightcone.engine.scratch` so on NERSC the - spill lands on Lustre instead of DVS-mounted home/CFS (where small- - file I/O is slow and can pressure the gateway nodes). - - *worker_image* is the registry ref the project's declared container - resolves to; the Gateway branch creates its cluster with exactly - this image (``None`` → the deployment's default). Ignored by the - other branches — they realize containers by wrapping recipes, not - via pod images. - - *max_workers* bounds the adaptive scaling of a Gateway cluster - (``lc run`` passes its job bound — there is never a reason to hold - more workers than dispatchable rules). Ignored everywhere else. - """ - if addr := os.environ.get("DASK_SCHEDULER_ADDRESS"): - if verbose: - print(f"→ Using existing Dask scheduler at {addr}") - yield {"DASK_SCHEDULER_ADDRESS": addr} - return - - if os.environ.get("DASK_GATEWAY__ADDRESS"): - with _gateway_cluster( - verbose=verbose, worker_image=worker_image, max_workers=max_workers - ) as name: - yield {GATEWAY_CLUSTER_ENV: name} - return - - if "SLURM_JOB_ID" in os.environ: - with _slurm_backed_cluster( - verbose=verbose, local_directory=local_directory - ) as addr: - yield {"DASK_SCHEDULER_ADDRESS": addr} - return - - with _local_cluster( - verbose=verbose, local_directory=local_directory - ) as addr: - yield {"DASK_SCHEDULER_ADDRESS": addr} - - -@contextmanager -def _gateway_cluster( - *, - verbose: bool, - worker_image: str | None, - max_workers: int | None, -) -> Iterator[str]: - """Create a run-scoped Dask Gateway cluster; yield its name. - - The Gateway client is configured entirely by ambient dask config — - on a lightcone JupyterHub deployment the ``DASK_GATEWAY__*`` env - vars carry the API address, the JupyterHub auth mode, and the proxy - address, so ``Gateway()`` needs no arguments here. - """ - from dask_gateway import Gateway - - gateway = Gateway() - # The server-declared cluster options (merged with ambient config - # defaults) tell us what this deployment exposes and what the - # effective worker shape/image will be. - try: - declared = dict(gateway.cluster_options()) - except Exception: - declared = {} - options: dict[str, object] = {} - if worker_image: - # ``image`` is a server-side cluster option declared by the - # deployment's options handler; a deployment that doesn't - # expose it rejects the request — surfaced below with guidance. - options["image"] = worker_image - if "environment" in declared: - # Self-provision everything our executor needs from a worker - # pod through the *standard* ``environment`` option, so the - # deployment's options handler can stay stock — no - # lightcone-specific injection required server-side. - base_env = dict(declared.get("environment") or {}) - options["environment"] = { - **base_env, - **_worker_environment(declared, worker_image), - } - try: - cluster = gateway.new_cluster(shutdown_on_close=True, **options) - except Exception as exc: - detail = ( - f" (requested image={worker_image!r} — if the deployment does " - "not expose an `image` cluster option, ask the hub admin to " - "add it to the gateway's cluster-options handler)" - if worker_image - else "" - ) - raise RuntimeError( - f"Could not create a Dask Gateway cluster ({exc}){detail}." - ) from exc - - bound = max(1, max_workers or 1) - if verbose: - image_note = f" with image {worker_image}" if worker_image else "" - print( - f"→ Created Dask Gateway cluster {cluster.name}{image_note}; " - f"scaling adaptively up to {bound} worker(s) " - f"(dashboard: {cluster.dashboard_link})" - ) - try: - cluster.adapt(minimum=1, maximum=bound) - client = cluster.get_client() - try: - _wait_first_worker(client, image=worker_image) - _assert_worker_resources(client) - finally: - client.close() - yield str(cluster.name) - finally: - # We created it, we cull it. shutdown() stops the cluster - # server-side; if lc dies before reaching this, shutdown_on_close - # and the deployment's idle timeout are the backstops. - try: - cluster.shutdown() - except Exception: - cluster.close() - if verbose: - print(f"→ Shut down Dask Gateway cluster {cluster.name}") - - -def _worker_environment( - declared: dict[str, object], worker_image: str | None -) -> dict[str, str]: - """Env vars lc provisions into scheduler/worker pods. - - Passed through the deployment's standard ``environment`` cluster - option — everything worker pods need that only lc (or the driver's - own environment) knows, keeping the deployment's options handler - free of lightcone-specific injection: - - * ``HOME``/``USER``/``LOGNAME`` — always set. Project images are - environment-agnostic (no passwd entry for the pod uid), so - ``getpass.getuser()`` (called by snakemake at startup) crashes - in a worker unless the env vars are present — and the notebook - pod may not export ``USER``/``LOGNAME`` itself (its own passwd - entry covers it), so the name is *derived* on the driver via - ``getpass`` rather than merely forwarded. Home paths are - identical on both sides by construction (same NFS mount). - * ``DASK_DISTRIBUTED__WORKER__RESOURCES__*`` — the scheduling - resource contract, mirrored from the deployment's declared - ``worker_cores``/``worker_memory`` option values. Dask matches - resource keys by exact presence; without these every rule hangs. - * ``LIGHTCONE_WORKER_IMAGE`` — the image the cluster runs (ours, - or the deployment default), recorded by the manifest layer as - execution ground truth. - """ - env: dict[str, str] = {} - if home := os.environ.get("HOME"): - env["HOME"] = home - try: - # Checks USER/LOGNAME/... env vars first, then the driver's - # passwd — one of the two works in any sane notebook pod. - user = getpass.getuser() - except (KeyError, OSError): - # Driver can't determine a name either; any stable non-empty - # value keeps snakemake alive, and jovyan is the Jupyter - # convention for uid 1000. - user = "jovyan" - env["USER"] = user - env["LOGNAME"] = user - - cores = declared.get("worker_cores") - if isinstance(cores, (int, float)) and cores > 0: - env["DASK_DISTRIBUTED__WORKER__RESOURCES__CPUS"] = str(int(cores)) - memory = declared.get("worker_memory") - if isinstance(memory, (int, float)) and memory > 0: - # Deployments conventionally declare worker_memory in GB - # (float); anything implausibly large for GB is already bytes. - mem_bytes = int(memory * 1e9) if memory < 1e6 else int(memory) - env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] = str(mem_bytes) - env["DASK_DISTRIBUTED__WORKER__RESOURCES__GPUS"] = str( - int(declared.get("worker_gpus") or 0) # type: ignore[call-overload] - ) - - image = worker_image or declared.get("image") - if isinstance(image, str) and image: - env["LIGHTCONE_WORKER_IMAGE"] = image - return env - - -def _wait_first_worker(client: object, *, image: str | None) -> None: - """Block until the created cluster has one live worker. - - An unpullable image or an unschedulable pool otherwise leaves the - run sitting at zero workers with no error at all — the classic - silent-hang failure mode. - """ - try: - timeout = int(os.environ.get(GATEWAY_WORKER_TIMEOUT_ENV) or 600) - except ValueError: - timeout = 600 - try: - client.wait_for_workers(n_workers=1, timeout=timeout) # type: ignore[attr-defined] - except Exception as exc: - image_hint = f"the worker image ({image or 'deployment default'}) cannot be pulled" - raise RuntimeError( - f"No Dask Gateway worker became ready within {timeout}s " - f"({exc}). Likely causes: {image_hint}, or the node pool " - "cannot schedule a worker (capacity/quota). Check the " - "JupyterLab Dask panel for the cluster's state, or raise " - f"{GATEWAY_WORKER_TIMEOUT_ENV}." - ) from exc - - -def _assert_worker_resources(client: object) -> None: - """Fail fast when Gateway workers don't advertise the resource contract. - - Dask schedules a task only on workers advertising *every* requested - resource key. The executor requests ``cpus`` for every rule and - ``memory`` for any rule with ``mem_mb``, so a deployment that forgot - to inject ``DASK_DISTRIBUTED__WORKER__RESOURCES__*`` into worker - pods makes every rule hang with no error — refuse loudly instead. - ``gpus`` is deliberately not required: a CPU-only deployment - legitimately omits it. + spilled task data and internal state files. ``lc materialize`` + resolves it to a path under :mod:`lightcone.engine.scratch`. """ - workers = client.scheduler_info().get("workers", {}) # type: ignore[attr-defined] - if not workers: - return - if any( - RESOURCE_CPUS in res and RESOURCE_MEMORY in res - for w in workers.values() - if (res := w.get("resources") or {}) is not None - ): - return - raise RuntimeError( - "Dask Gateway workers do not advertise the lightcone resource " - f"contract ({RESOURCE_CPUS}+{RESOURCE_MEMORY}, plus " - f"{RESOURCE_GPUS} on GPU pools); per-rule resource requests " - "would never schedule. lc provisions these via the gateway's " - "`environment` cluster option — this deployment likely doesn't " - "expose that option (or strips it); ask the hub admin to expose " - "the standard image/worker_cores/worker_memory/environment " - "options." - ) - - -@contextmanager -def _local_cluster( - *, verbose: bool, local_directory: str | None -) -> Iterator[str]: from dask.distributed import LocalCluster shape = _detect_node_shape() - # Workers must advertise every key the executor may request — Dask - # matches by exact key presence — or rules with ``mem_mb`` / - # ``gpus_per_task`` would never schedule on a workstation. cluster = LocalCluster( n_workers=1, threads_per_worker=shape.cpus, @@ -405,112 +96,6 @@ def _local_cluster( f"scheduler at {cluster.scheduler_address}" ) try: - yield cluster.scheduler_address - finally: - cluster.close() - - -@contextmanager -def _slurm_backed_cluster( - *, verbose: bool, local_directory: str | None -) -> Iterator[str]: - from dask.distributed import LocalCluster - - if shutil.which("dask") is None: - raise RuntimeError( - "`dask` CLI is not on PATH inside the SLURM allocation. " - "Install lightcone-cli (and its `distributed` dep) into the " - "environment activated by your sbatch/salloc." - ) - - shape = _detect_node_shape() - nnodes = int(os.environ.get("SLURM_NNODES") or 1) - - # Default LocalCluster binds the scheduler to 127.0.0.1, which workers - # on remote nodes cannot reach. Bind to the driver's hostname so srun- - # launched workers across the allocation can connect. SLURMD_NODENAME - # is the SLURM-canonical name; gethostname() is a sane fallback. - scheduler_host = os.environ.get("SLURMD_NODENAME") or socket.gethostname() - cluster = LocalCluster( - n_workers=0, - host=scheduler_host, - dashboard_address=":0", - local_directory=local_directory, - silence_logs=logging.INFO if verbose else logging.WARNING, - ) - addr = cluster.scheduler_address - - if verbose: - print( - f"→ SLURM allocation detected ({nnodes} node(s), " - f"{shape.cpus} cpu/node, {shape.gpus} gpu/node); " - f"launching workers via srun. Scheduler: {addr}" - ) - - worker_cmd = [ - "srun", - f"--ntasks={nnodes}", - "--ntasks-per-node=1", - "dask", - "worker", - addr, - "--nthreads", - str(shape.cpus), - "--nworkers", - "1", - "--resources", - _resources_arg(shape), - "--no-dashboard", - # Each srun task is a single run-scoped worker; an auto-restart - # nanny adds no value (srun won't relaunch the task either) and - # logs "Worker process died unexpectedly" when retire_workers - # asks the worker to exit on shutdown. - "--no-nanny", - ] - if local_directory: - worker_cmd.extend(["--local-directory", local_directory]) - # Hide the worker's INFO-level connection chatter (Nanny start, - # scheduler registration, etc.) — useful only when debugging the - # cluster itself. WARNING+ still surface real issues. The newer - # `dask worker` CLI dropped `--silence-logs`, so we drive it via - # Dask's config env var instead; srun inherits env by default. - worker_env = dict(os.environ) - if not verbose: - worker_env.setdefault("DASK_LOGGING__DISTRIBUTED", "warning") - workers = subprocess.Popen(worker_cmd, env=worker_env) - - try: - from dask.distributed import Client - - client = Client(addr) - try: - client.wait_for_workers(n_workers=nnodes, timeout=120) - if verbose: - print(f"→ {nnodes} dask worker(s) registered.") - finally: - client.close() - yield addr + yield {"DASK_SCHEDULER_ADDRESS": cluster.scheduler_address} finally: - # Graceful shutdown: ask the scheduler to retire workers so each - # `dask worker` process exits on its own. srun then sees its task - # exit with code 0 and terminates silently. SIGTERM-ing srun - # directly (the prior path) prints "srun: forcing job - # termination" / "task 0: Killed" to stderr on every clean run. - try: - client = Client(addr, timeout="10s") - try: - client.retire_workers(close_workers=True, remove=True) - finally: - client.close() - except Exception: - pass - try: - workers.wait(timeout=20) - except subprocess.TimeoutExpired: - workers.terminate() - try: - workers.wait(timeout=10) - except subprocess.TimeoutExpired: - workers.kill() - workers.wait() cluster.close() diff --git a/src/lightcone/engine/environment.py b/src/lightcone/engine/environment.py new file mode 100644 index 00000000..4348c367 --- /dev/null +++ b/src/lightcone/engine/environment.py @@ -0,0 +1,296 @@ +"""The project environment model: mode, identity, and the lock scan. + +The environment is ``pyproject.toml`` + ``uv.lock`` + ``.python-version`` +— uv is the only substrate. Everything here is derived from those repo +files plus the closed ``[tool.lightcone]`` surface: + +* **Mode** is derived, never configured: declaring + ``[tool.lightcone.image]`` (or shipping ``Containerfile.extra``) *is* + the escalation into containerized mode. +* **``env_version``** is the environment identity — one formula for + both modes (direct mode hashes empty image fields). It sits inside + every output's ``code_version``, so an environment edit stales + exactly the outputs whose semantics it could change: all of them. +* **The lock scan** refuses what cannot be audited (path/editable + dependencies other than the project's own package) and reports what + weakens identity (registry sdists built locally). +""" +from __future__ import annotations + +import hashlib +import tomllib +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Any + +from lightcone.engine.image.declaration import ( + EMPTY_CANONICAL_JSON, + ImageDeclaration, + load_image_declaration, +) +from lightcone.engine.manifest import canonical_json, frame + + +class ProjectEnvironmentError(Exception): + """A project-environment problem the user must fix (rendered as a + clean CLI error, never a traceback).""" + + +class Mode(StrEnum): + DIRECT = "direct" + CONTAINERIZED = "containerized" + + +#: The closed, audited list of uv install-selection settings that flow +#: into ``env_version`` (v4's list, carried by spec §3): anything that +#: changes *which artifacts* ``uv sync`` materializes from the same +#: lock. +_INSTALL_SETTING_KEYS = ( + "default-groups", + "no-binary", + "no-binary-package", + "no-build", + "no-build-package", + "config-settings", + "no-build-isolation", + "no-build-isolation-package", +) + +#: Valid sub-tables of ``[tool.lightcone]`` — a closed surface, like the +#: image table itself. +_LIGHTCONE_KEYS = {"image", "sandbox"} +_SANDBOX_KEYS = {"writable-project"} + + +@dataclass(frozen=True) +class InstallSettings: + """Normalized install-selection settings from ``[tool.uv]``.""" + + values: tuple[tuple[str, str], ...] # (key, canonical-JSON value) pairs + + @classmethod + def from_tool_uv(cls, tool_uv: dict[str, Any]) -> InstallSettings: + pairs = [] + for key in _INSTALL_SETTING_KEYS: + pairs.append((key, canonical_json(tool_uv.get(key)))) + return cls(values=tuple(pairs)) + + def canonical_json(self) -> str: + return canonical_json(dict(self.values)) + + +@dataclass(frozen=True) +class EnvironmentSpec: + """The loaded, validated project environment.""" + + root: Path + mode: Mode + python_version: str + image: ImageDeclaration | None # None ⇔ direct mode + env_version: str # "sha256:" + writable_project_outputs: frozenset[str] + + @property + def venv(self) -> Path: + return self.root / ".venv" + + +def load_environment(root: Path) -> EnvironmentSpec: + """Single parse point for the project environment. + + Raises :class:`ProjectEnvironmentError` on missing environment + files or banned states (authored root Containerfile; packaged + project in containerized mode). + """ + root = root.resolve() + + if (root / "Containerfile").is_file(): + raise ProjectEnvironmentError( + f"{root}/Containerfile: v6 generates images from the lock — " + "an authored root Containerfile is not consumed by anything " + "and would mislead readers. Delete or rename it; declare " + "system dependencies in [tool.lightcone.image] instead." + ) + + pyproject_path = root / "pyproject.toml" + if not pyproject_path.is_file(): + raise ProjectEnvironmentError( + f"{root}: no pyproject.toml — the environment is " + "pyproject.toml + uv.lock + .python-version. Run `lc init` " + "to scaffold it." + ) + try: + pyproject = tomllib.loads(pyproject_path.read_text()) + except tomllib.TOMLDecodeError as e: + raise ProjectEnvironmentError(f"{pyproject_path}: invalid TOML: {e}") from e + + pv_path = root / ".python-version" + if not pv_path.is_file(): + raise ProjectEnvironmentError( + f"{root}: no .python-version — the exact interpreter pin is " + "part of the environment identity. Run `lc init` to scaffold it." + ) + python_version = pv_path.read_text().strip() + if not python_version: + raise ProjectEnvironmentError(f"{pv_path}: empty .python-version file.") + + if not (root / "uv.lock").is_file(): + raise ProjectEnvironmentError( + f"{root}: no uv.lock — run `uv lock` (or `lc init`) to lock " + "the environment." + ) + + tool_lightcone = pyproject.get("tool", {}).get("lightcone", {}) + if not isinstance(tool_lightcone, dict): + raise ProjectEnvironmentError("[tool.lightcone] must be a table.") + if unknown := set(tool_lightcone) - _LIGHTCONE_KEYS: + raise ProjectEnvironmentError( + f"[tool.lightcone]: unknown key(s) " + f"{', '.join(sorted(repr(k) for k in unknown))}; valid: " + f"{', '.join(sorted(_LIGHTCONE_KEYS))}." + ) + + image = load_image_declaration(root, pyproject) + mode = Mode.CONTAINERIZED if image is not None else Mode.DIRECT + packaged = "build-system" in pyproject + + if packaged and mode is Mode.CONTAINERIZED: + raise ProjectEnvironmentError( + "containerized mode requires a virtual project (no " + "[build-system] in pyproject.toml): the image is built " + "--no-install-project — code never enters an image — so a " + "packaged project's own import would fail inside its " + "container. Restructure as a virtual project." + ) + + install_settings = InstallSettings.from_tool_uv( + pyproject.get("tool", {}).get("uv", {}) or {} + ) + + env_version = compute_env_version( + uv_lock_bytes=(root / "uv.lock").read_bytes(), + python_version_bytes=pv_path.read_bytes(), + install_settings=install_settings, + image=image, + ) + + return EnvironmentSpec( + root=root, + mode=mode, + python_version=python_version, + image=image, + env_version=env_version, + writable_project_outputs=_writable_project_outputs(tool_lightcone), + ) + + +def _writable_project_outputs(tool_lightcone: dict[str, Any]) -> frozenset[str]: + sandbox = tool_lightcone.get("sandbox", {}) + if not isinstance(sandbox, dict): + raise ProjectEnvironmentError("[tool.lightcone.sandbox] must be a table.") + if unknown := set(sandbox) - _SANDBOX_KEYS: + raise ProjectEnvironmentError( + f"[tool.lightcone.sandbox]: unknown key(s) " + f"{', '.join(sorted(repr(k) for k in unknown))}; valid: " + f"{', '.join(sorted(_SANDBOX_KEYS))}." + ) + raw = sandbox.get("writable-project", []) + if not isinstance(raw, list) or not all(isinstance(o, str) for o in raw): + raise ProjectEnvironmentError( + "[tool.lightcone.sandbox] writable-project must be a list of " + "output ids." + ) + return frozenset(raw) + + +def compute_env_version( + *, + uv_lock_bytes: bytes, + python_version_bytes: bytes, + install_settings: InstallSettings, + image: ImageDeclaration | None, +) -> str: + """The environment identity (spec §3) — one formula for both modes. + + ``sha256(uv.lock bytes ‖ .python-version bytes ‖ canonical + install-settings JSON ‖ canonical image-declaration JSON ‖ + Containerfile.extra sha-or-null)``, length-framed. Direct mode + hashes the empty image shape and a null extra. + """ + h = hashlib.sha256() + frame(h, "uv.lock", uv_lock_bytes) + frame(h, "python-version", python_version_bytes) + frame(h, "install-settings", install_settings.canonical_json().encode("utf-8")) + image_json = image.canonical_json() if image else EMPTY_CANONICAL_JSON + frame(h, "image", image_json.encode("utf-8")) + extra = image.extra_sha256 if image and image.extra_sha256 else "null" + frame(h, "containerfile-extra", extra.encode("utf-8")) + return f"sha256:{h.hexdigest()}" + + +# --------------------------------------------------------------------------- +# Lock scan +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class LockScan: + """What the lock says about auditability.""" + + refusals: tuple[str, ...] # path/directory/editable deps ≠ own package + sdist_built: tuple[str, ...] # registry packages with no wheel at all + non_default_groups: tuple[str, ...] # advisory: outside lc's guarantees + + +def scan_lock(root: Path) -> LockScan: + """Scan ``uv.lock`` + ``pyproject.toml`` for identity hazards. + + * **Refusal**: path / directory / editable dependencies other than + the project's own package — unauditable inputs (their bytes are + not pinned by the lock). + * **Report**: registry packages shipping no wheel (the sdist is + built locally at sync time — identity covers the sdist, not the + build). + * **Advisory**: dependency groups beyond uv's default — installable + states the identity does not cover. + """ + lock_path = root / "uv.lock" + try: + lock = tomllib.loads(lock_path.read_text()) + except (OSError, tomllib.TOMLDecodeError) as e: + raise ProjectEnvironmentError(f"{lock_path}: unreadable: {e}") from e + + pyproject = tomllib.loads((root / "pyproject.toml").read_text()) + own_name = pyproject.get("project", {}).get("name") + + refusals: list[str] = [] + sdist_built: list[str] = [] + for pkg in lock.get("package", []): + name = pkg.get("name", "?") + source = pkg.get("source", {}) or {} + if any(k in source for k in ("path", "directory", "editable")): + if name != own_name: + refusals.append( + f"{name}: {next(k for k in ('path', 'directory', 'editable') if k in source)} " + "dependency — unauditable (bytes not pinned by the lock)" + ) + continue + if "virtual" in source: + continue + if "registry" in source and "sdist" in pkg and not pkg.get("wheels"): + sdist_built.append(name) + + groups = set(pyproject.get("dependency-groups", {}) or {}) + tool_uv = pyproject.get("tool", {}).get("uv", {}) or {} + default_groups = tool_uv.get("default-groups", ["dev"]) + if default_groups == "all": + non_default: set[str] = set() + else: + non_default = groups - set(default_groups) + + return LockScan( + refusals=tuple(sorted(refusals)), + sdist_built=tuple(sorted(sdist_built)), + non_default_groups=tuple(sorted(non_default)), + ) diff --git a/src/lightcone/engine/image/__init__.py b/src/lightcone/engine/image/__init__.py new file mode 100644 index 00000000..d65c6105 --- /dev/null +++ b/src/lightcone/engine/image/__init__.py @@ -0,0 +1,167 @@ +"""Content-addressed container images for containerized-mode projects. + +The internal API is Modal-inspired: a typed, immutable +:class:`~lightcone.engine.image.definition.ImageDefinition` assembled +from the one-TOML-table user surface, rendered deterministically to +Containerfile text, with identity a pure function of the definition and +pluggable builder/runtime backends (podman today). + +This module is the only import surface for the CLI, launcher, and +status layers. +""" +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from lightcone.engine import lc_version +from lightcone.engine.image import constants +from lightcone.engine.image.definition import ImageDefinition +from lightcone.engine.image.errors import ( + DeclarationError, + ImageError, + ImageMissingError, + PodmanUnavailableError, +) +from lightcone.engine.image.identity import EnvInputs, compute_tag +from lightcone.engine.image.record import ( + BuildRecord, + read_record, + snapshot_sha256, + write_record, +) +from lightcone.engine.image.render import render + +if TYPE_CHECKING: + from lightcone.engine.environment import EnvironmentSpec + from lightcone.engine.image.builder import Builder + + +def _definition(env: EnvironmentSpec) -> ImageDefinition: + if env.image is None: + raise DeclarationError( + "direct-mode project has no image — declare " + "[tool.lightcone.image] to containerize." + ) + return ImageDefinition.from_declaration( + env.image, + env_version=env.env_version, + python_version=env.python_version, + ) + + +def ensure_image( + project: Path, + env: EnvironmentSpec, + *, + force: bool = False, + builder: Builder | None = None, + on_progress: Callable[[str], None] = lambda _: None, +) -> BuildRecord: + """Declaration → definition → render → tag → (build if absent) → + record. Idempotent: a tag hit is a no-op (spec §3).""" + from lightcone.engine.image.builder import BuildContext + from lightcone.engine.image.builder_podman import PodmanBuilder + + defn = _definition(env) + rendered = render(defn) + inputs = EnvInputs.read(project) + tag = compute_tag(rendered, inputs) + + b = builder or PodmanBuilder() + existing = read_record(project) + if not force and existing and existing.tag == tag and b.exists(tag): + return existing + + on_progress( + f"building {tag} — first run after an environment change; ~minutes" + ) + context = BuildContext(containerfile_text=rendered.text, inputs=inputs) + result = b.build(context, tag=tag) + record = BuildRecord( + tag=result.tag, + image_id=result.image_id, + digest=result.digest, + platform=result.platform, + env_version=env.env_version, + lc_version=lc_version(), + base=str(defn.base), + built_at=datetime.now(UTC).isoformat(timespec="seconds"), + dpkg_snapshot_sha256=snapshot_sha256(result.dpkg_snapshot_text), + ) + write_record(project, record, result.dpkg_snapshot_text) + return record + + +def resolve_pinned( + project: Path, + env: EnvironmentSpec, + *, + builder: Builder | None = None, +) -> BuildRecord: + """Resolve the current environment to its build record, verifying + the local store still holds the recorded image *id* (execution pins + by id, so a retagged store can never substitute — the missing-image + error is the only failure mode). ``lc run`` never builds — the + message embeds the exact command.""" + from lightcone.engine.image.builder_podman import PodmanBuilder + + tag = compute_tag(render(_definition(env)), EnvInputs.read(project)) + record = read_record(project) + b = builder or PodmanBuilder() + if record is None or record.tag != tag or not b.exists(record.image_id): + raise ImageMissingError( + f"the environment image {tag} is not built — run: lc build" + ) + return record + + +@dataclass(frozen=True) +class ImageStatus: + tag: str + built: bool + image_id: str | None + + +def image_status( + project: Path, + env: EnvironmentSpec, + *, + builder: Builder | None = None, +) -> ImageStatus: + """The ``lc status`` header's image line — offline and local-only + (reads the build record and the local image store, never the + network).""" + tag = compute_tag(render(_definition(env)), EnvInputs.read(project)) + record = read_record(project) + built = False + if record is not None and record.tag == tag: + try: + from lightcone.engine.image.builder_podman import PodmanBuilder + + b = builder or PodmanBuilder() + built = b.exists(tag) + except PodmanUnavailableError: + built = False + return ImageStatus( + tag=tag, + built=built, + image_id=record.image_id if built and record else None, + ) + + +__all__ = [ + "BuildRecord", + "ImageError", + "ImageMissingError", + "ImageStatus", + "PodmanUnavailableError", + "constants", + "ensure_image", + "image_status", + "read_record", + "resolve_pinned", +] diff --git a/src/lightcone/engine/image/builder.py b/src/lightcone/engine/image/builder.py new file mode 100644 index 00000000..b408611d --- /dev/null +++ b/src/lightcone/engine/image/builder.py @@ -0,0 +1,53 @@ +"""Builder protocol + the three-file build context. + +:class:`BuildContext` is the structural guarantee behind G5: there is +no code path that can put project code into an image — the context is +*exactly* the rendered Containerfile, ``pyproject.toml``, and +``uv.lock``, staged world-readable into a fresh directory. +""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from lightcone.engine.image.identity import EnvInputs + + +@dataclass(frozen=True) +class BuildContext: + containerfile_text: str + #: The exact bytes the tag was computed over — staged verbatim, so + #: the built image can never diverge from the hashed identity. + inputs: EnvInputs + + def stage(self, into: Path) -> Path: + """Write the three files into *into*; returns the Containerfile + path. World-readable: the build may run in a user namespace with + a different effective uid.""" + into.mkdir(parents=True, exist_ok=True) + containerfile = into / "Containerfile" + containerfile.write_text(self.containerfile_text) + (into / "pyproject.toml").write_bytes(self.inputs.pyproject_bytes) + (into / "uv.lock").write_bytes(self.inputs.uv_lock_bytes) + for p in into.iterdir(): + p.chmod(0o644) + return containerfile + + +@dataclass(frozen=True) +class BuildResult: + tag: str + image_id: str + digest: str | None + platform: str + dpkg_snapshot_text: str + + +class Builder(Protocol): + """A backend that can materialize a rendered image (podman today; + remote builders return behind this same protocol).""" + + def exists(self, tag: str) -> bool: ... + + def build(self, context: BuildContext, *, tag: str) -> BuildResult: ... diff --git a/src/lightcone/engine/image/builder_podman.py b/src/lightcone/engine/image/builder_podman.py new file mode 100644 index 00000000..f14037c3 --- /dev/null +++ b/src/lightcone/engine/image/builder_podman.py @@ -0,0 +1,146 @@ +"""The podman builder: build invocation and pointed error mapping. + +Every failure class surfaces as a specific error with an actionable +message — never a raw build log (the Modal lesson): the generated +contract-check layer's distinct exit codes map to +:class:`BaseContractError`, apt's "Unable to locate package" maps to +:class:`AptPackageNotFoundError`, a manifest-list architecture miss +maps to the platform contract, and anything else carries a bounded log +tail. +""" +from __future__ import annotations + +import re +import subprocess +import tempfile +from collections import deque +from pathlib import Path + +from lightcone.engine.image import constants +from lightcone.engine.image.builder import BuildContext, BuildResult +from lightcone.engine.image.errors import ( + AptPackageNotFoundError, + BaseContractError, + ImageBuildError, + require_podman, +) + +_APT_NOT_FOUND_RE = re.compile(r"E: Unable to locate package (\S+)") +_EXIT_STATUS_RE = re.compile(r"exit (?:status|code):? (\d+)") +_ARCH_MISS_RE = re.compile(r"no image found in manifest list for architecture") + +_LOG_TAIL_LINES = 60 + +_CONTRACT_MESSAGES = { + constants.EXIT_NO_SH: ( + "the base image has no POSIX shell at /bin/sh — build stages run " + "through it. Use a base that provides one." + ), + constants.EXIT_MUSL_BASE: ( + "the base image is musl-based (Alpine?) — manylinux wheels and " + "uv-managed interpreters require glibc. Use a glibc base " + "(the default Debian base, or a Debian/Ubuntu-family ref)." + ), + constants.EXIT_NO_APT: ( + "the base image has no apt, but system-packages are declared. " + "Two escapes: use a Debian/Ubuntu-family base, or move the " + "install into Containerfile.extra." + ), +} + + +class PodmanBuilder: + def __init__(self, podman: str = "podman") -> None: + require_podman(podman) + self._podman = podman + + def exists(self, tag: str) -> bool: + proc = subprocess.run( + [self._podman, "image", "exists", tag], + capture_output=True, + check=False, + ) + return proc.returncode == 0 + + def build(self, context: BuildContext, *, tag: str) -> BuildResult: + with tempfile.TemporaryDirectory(prefix="lc-image-") as staged: + containerfile = context.stage(Path(staged)) + proc = subprocess.run( + [ + self._podman, "build", + "--file", str(containerfile), + "--tag", tag, + staged, + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + self._raise_mapped(proc.stdout, proc.stderr) + + image_id, digest, platform = self._inspect(tag) + snapshot = self._capture_snapshot(tag) + return BuildResult( + tag=tag, + image_id=image_id, + digest=digest, + platform=platform, + dpkg_snapshot_text=snapshot, + ) + + def _raise_mapped(self, stdout: str, stderr: str) -> None: + combined = stdout + "\n" + stderr + if m := _APT_NOT_FOUND_RE.search(combined): + raise AptPackageNotFoundError(m.group(1)) + if _ARCH_MISS_RE.search(combined): + raise BaseContractError( + "the declared base provides no image for this architecture " + "— the base contract requires linux/amd64 (linux/arm64 is " + "used on Apple silicon when available)." + ) + if m := _EXIT_STATUS_RE.search(combined): + code = int(m.group(1)) + if code in _CONTRACT_MESSAGES: + raise BaseContractError(_CONTRACT_MESSAGES[code]) + tail = "\n".join( + deque((stdout + stderr).splitlines(), maxlen=_LOG_TAIL_LINES) + ) + raise ImageBuildError(f"podman build failed:\n{tail}") + + def _inspect(self, tag: str) -> tuple[str, str | None, str]: + proc = subprocess.run( + [ + self._podman, "image", "inspect", tag, + "--format", "{{.Id}}|{{.Digest}}|{{.Os}}/{{.Architecture}}", + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + raise ImageBuildError( + f"podman image inspect {tag} failed after a successful " + f"build:\n{proc.stderr.strip()}" + ) + image_id, digest, platform = proc.stdout.strip().split("|", 2) + if not image_id.startswith("sha256:"): + image_id = f"sha256:{image_id}" + return image_id, (digest or None), platform + + def _capture_snapshot(self, tag: str) -> str: + """Read the baked dpkg snapshot (taken in the final build stage, + after any extra stage, so it attests everything installed).""" + proc = subprocess.run( + [ + self._podman, "run", "--rm", "--pull=never", "--net=none", + "--entrypoint=", tag, + "cat", constants.DPKG_SNAPSHOT_PATH, + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + return "snapshot unavailable" + return proc.stdout diff --git a/src/lightcone/engine/image/constants.py b/src/lightcone/engine/image/constants.py new file mode 100644 index 00000000..c3c14032 --- /dev/null +++ b/src/lightcone/engine/image/constants.py @@ -0,0 +1,72 @@ +"""Engine constants for the image layer. + +These digests ship inside the locked engine, so new constants reach a +project only through an engine release + relock — the image tag and +``env_version`` move together (spec §3). They are never resolved at +run time. + +Bump procedure: resolve the current manifest-*list* digests (they must +cover linux/amd64 and linux/arm64 so the rendered Containerfile text — +and therefore the tag — stays architecture-independent) and paste them +here, e.g.:: + + podman manifest inspect docker.io/library/debian@sha256: + # must report an OCI image index with amd64 + arm64 entries + +Honest residue, documented: ``uv python install`` fetches a +python-build-standalone interpreter pinned by (uv version, +``.python-version``), not by a digest we hold — attestation-grade +interpreter identity is recorded in the manifest (``python_build``), +digest-pinning it is a hardening candidate alongside apt snapshot +pinning. +""" +from __future__ import annotations + +from lightcone.engine.uv_env import OFFLINE_OVERLAY as _OFFLINE_OVERLAY + +#: Default base image (used when ``[tool.lightcone.image]`` declares no +#: ``base``). Digest is the manifest-LIST digest — one spelling covers +#: amd64 and arm64. +DEFAULT_BASE_NAME = "docker.io/library/debian:bookworm-slim" +DEFAULT_BASE_DIGEST = ( + "sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241" +) + +#: The pinned uv distribution: the official uv image, copied from by +#: digest (``COPY --from=ghcr.io/astral-sh/uv@ /uv …``). +#: Manifest-list digest, same arch-independence rationale. +UV_VERSION = "0.12.3" +UV_IMAGE = "ghcr.io/astral-sh/uv" +UV_IMAGE_DIGEST = ( + "sha256:2d890623d310b57771ce840f0da5eed5fc6d657da05ffaa45d82797b53fa3abc" +) + +#: Exact interpreter patch scaffolded into ``.python-version`` by +#: ``lc init`` (projects may pin a different one; identity follows the +#: file, not this constant). +DEFAULT_PYTHON = "3.12.12" + +#: In-image filesystem layout. ``/opt`` because the base contract +#: guarantees nothing about the base beyond an OS layer — these paths +#: are lc's own namespace. +OPT_PYTHON = "/opt/python" +OPT_VENV = "/opt/venv" +LC_DIR = "/opt/lc" +UV_BIN = f"{LC_DIR}/bin/uv" +PROJECT_STAGE_DIR = f"{LC_DIR}/project" +DPKG_SNAPSHOT_PATH = f"{LC_DIR}/dpkg-snapshot.txt" +IDENTITY_PATH = f"{LC_DIR}/identity.json" + +#: Contract-check exit codes emitted by the generated Containerfile's +#: check layer; the builder maps them back to BaseContractError. +EXIT_NO_SH = 41 +EXIT_MUSL_BASE = 43 +EXIT_NO_APT = 44 + +#: The offline overlay baked into the image's FINAL stage only — the +#: build's own ``uv sync`` layer must keep network (spec §11 step 6; +#: ordering pinned by golden tests). Derived from the worker-exec +#: overlay so the direct and containerized offline postures can never +#: drift; ``UV_NO_SYNC`` is image-only (nothing inside an image ever +#: syncs). +OFFLINE_ENV = {**_OFFLINE_OVERLAY, "UV_NO_SYNC": "1"} diff --git a/src/lightcone/engine/image/declaration.py b/src/lightcone/engine/image/declaration.py new file mode 100644 index 00000000..c85fbe8d --- /dev/null +++ b/src/lightcone/engine/image/declaration.py @@ -0,0 +1,191 @@ +"""Parse and validate the container-hatch declaration surface. + +The *entire* user-facing surface of the container hatch is one TOML +table in ``pyproject.toml`` plus an optional ``Containerfile.extra``:: + + [tool.lightcone.image] + base = "nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:9f2c…" + system-packages = ["texlive-latex-base", "r-base-core"] + +Its presence (or the extra file's) IS the escalation into containerized +mode — there is no separate switch. Every key is hashed into +``env_version`` and the image tag, which is why the surface is closed: +an unknown key is a refusal, never silently ignored. + +Static refusals live here (parse time — they fire on every verb, long +before podman is involved). Contract properties that depend on the +base image's *contents* (musl, apt presence) are build-time checks in +the generated Containerfile. +""" +from __future__ import annotations + +import hashlib +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path + +from lightcone.engine.image.errors import DeclarationError +from lightcone.engine.manifest import canonical_json + +EXTRA_FILENAME = "Containerfile.extra" + +_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +#: Debian package-name grammar (policy §5.6.1): lowercase alphanumerics +#: plus ``+ - .``, at least two characters, starting alphanumeric. +_APT_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9+.-]+$") +_VALID_KEYS = {"base", "system-packages"} + + +@dataclass(frozen=True) +class BaseRef: + """A digest-pinned OCI reference.""" + + name: str # e.g. "nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04" + digest: str # "sha256:<64 hex>" + + def __str__(self) -> str: + return f"{self.name}@{self.digest}" + + @classmethod + def parse(cls, ref: str) -> BaseRef: + name, sep, digest = ref.partition("@") + if not sep or not name: + raise DeclarationError( + f"[tool.lightcone.image] base = {ref!r}: pin the digest — " + "`base` must be a digest-pinned OCI ref (`name@sha256:…`); " + "a tag-only ref makes the image identity a function of " + "registry state instead of the repo." + ) + if not _DIGEST_RE.match(digest): + raise DeclarationError( + f"[tool.lightcone.image] base = {ref!r}: {digest!r} is not " + "a valid digest (expected `sha256:` + 64 hex characters)." + ) + return cls(name=name, digest=digest) + + +@dataclass(frozen=True) +class ImageDeclaration: + """The parsed, validated container-hatch declaration.""" + + base: BaseRef | None # None ⇒ the engine's default base + system_packages: tuple[str, ...] # sorted, deduped + extra: str | None # Containerfile.extra content, verbatim + extra_sha256: str | None + + def canonical_json(self) -> str: + """Canonical serialization hashed into ``env_version``. + + Direct-mode projects hash the same shape with empty fields, so + the ``env_version`` formula stays one formula (spec §3) — + callers with no declaration use :data:`EMPTY_CANONICAL_JSON`. + """ + return canonical_json( + { + "base": str(self.base) if self.base else None, + "system-packages": list(self.system_packages), + } + ) + + +#: What a direct-mode project hashes in the image-declaration slot. +EMPTY_CANONICAL_JSON = canonical_json({"base": None, "system-packages": []}) + + +def load_image_declaration( + project: Path, pyproject: dict[str, object] | None = None +) -> ImageDeclaration | None: + """Parse the project's image declaration; ``None`` ⇔ direct mode. + + Containerized mode is derived, never configured: the presence of + the ``[tool.lightcone.image]`` table (even empty) OR a + ``Containerfile.extra`` file is the escalation. *pyproject* accepts + an already-parsed ``pyproject.toml`` dict (the environment loader + passes its own parse through). + + Raises :class:`DeclarationError` on any static violation. + """ + table: dict[str, object] | None = None + if pyproject is None: + pyproject_path = project / "pyproject.toml" + if pyproject_path.is_file(): + try: + pyproject = tomllib.loads(pyproject_path.read_text()) + except tomllib.TOMLDecodeError as e: + raise DeclarationError( + f"{pyproject_path}: invalid TOML: {e}" + ) from e + if pyproject is not None: + tool = pyproject.get("tool", {}) + raw = tool.get("lightcone", {}).get("image") if isinstance(tool, dict) else None + if raw is not None: + if not isinstance(raw, dict): + raise DeclarationError( + "[tool.lightcone.image] must be a table." + ) + table = raw + + extra_path = project / EXTRA_FILENAME + extra: str | None = None + if extra_path.is_file(): + extra = extra_path.read_text() + _validate_extra(extra) + + if table is None and extra is None: + return None + + table = table or {} + if unknown := set(table) - _VALID_KEYS: + raise DeclarationError( + f"[tool.lightcone.image]: unknown key(s) " + f"{', '.join(sorted(repr(k) for k in unknown))}. The surface " + f"is closed (every key is hashed into the environment " + f"identity); valid keys: base, system-packages." + ) + + base: BaseRef | None = None + if (base_raw := table.get("base")) is not None: + if not isinstance(base_raw, str): + raise DeclarationError("[tool.lightcone.image] base must be a string.") + base = BaseRef.parse(base_raw) + + packages_raw = table.get("system-packages", []) + if not isinstance(packages_raw, list) or not all( + isinstance(p, str) for p in packages_raw + ): + raise DeclarationError( + "[tool.lightcone.image] system-packages must be a list of " + "apt package names." + ) + for pkg in packages_raw: + if not _APT_NAME_RE.match(pkg): + raise DeclarationError( + f"[tool.lightcone.image] system-packages: {pkg!r} is not a " + "valid apt package name (lowercase alphanumerics plus " + "'+', '-', '.'; unsure of the name? try: " + f"`apt-cache search {pkg.lower()}`)." + ) + system_packages = tuple(sorted(set(packages_raw))) + + return ImageDeclaration( + base=base, + system_packages=system_packages, + extra=extra, + extra_sha256=( + hashlib.sha256(extra.encode("utf-8")).hexdigest() + if extra is not None + else None + ), + ) + + +def _validate_extra(extra: str) -> None: + for line in extra.splitlines(): + if line.strip().upper().startswith("FROM "): + raise DeclarationError( + f"{EXTRA_FILENAME}: contains a FROM line. The extra stage " + "is generated `FROM` the derived environment image — " + "write build instructions only (RUN, ENV, COPY …); the " + "generator owns the stage structure." + ) diff --git a/src/lightcone/engine/image/definition.py b/src/lightcone/engine/image/definition.py new file mode 100644 index 00000000..6ef34da8 --- /dev/null +++ b/src/lightcone/engine/image/definition.py @@ -0,0 +1,71 @@ +"""The image definition — a pure value object the generator renders. + +Modal-style internal API: an :class:`ImageDefinition` is assembled from +the project's declaration plus engine constants, and everything +downstream (rendered Containerfile text, tag, build) is a pure function +of it. The layering is **fixed by the generator, never user-ordered** +(spec §2): base → contract checks → apt → pinned uv → interpreter → +locked sync → extra stage → final ENV contract. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from lightcone.engine.image import constants +from lightcone.engine.image.declaration import BaseRef, ImageDeclaration + + +@dataclass(frozen=True) +class UvDistribution: + """The pinned uv binary source (engine constant).""" + + version: str + image: str + image_digest: str + + @property + def copy_from_ref(self) -> str: + return f"{self.image}@{self.image_digest}" + + +UV_DIST = UvDistribution( + version=constants.UV_VERSION, + image=constants.UV_IMAGE, + image_digest=constants.UV_IMAGE_DIGEST, +) + +DEFAULT_BASE = BaseRef( + name=constants.DEFAULT_BASE_NAME, digest=constants.DEFAULT_BASE_DIGEST +) + + +@dataclass(frozen=True) +class ImageDefinition: + """Everything that determines the rendered Containerfile.""" + + base: BaseRef + system_packages: tuple[str, ...] + python_version: str # exact patch, from .python-version + uv: UvDistribution + extra_stage: str | None # Containerfile.extra content, verbatim + env_version: str # baked as a LABEL + /opt/lc/identity.json + + @classmethod + def from_declaration( + cls, + declaration: ImageDeclaration, + *, + env_version: str, + python_version: str, + ) -> ImageDefinition: + """*python_version* comes from the loaded + :class:`~lightcone.engine.environment.EnvironmentSpec` — the + environment layer already read and validated the pin.""" + return cls( + base=declaration.base or DEFAULT_BASE, + system_packages=declaration.system_packages, + python_version=python_version, + uv=UV_DIST, + extra_stage=declaration.extra, + env_version=env_version, + ) diff --git a/src/lightcone/engine/image/errors.py b/src/lightcone/engine/image/errors.py new file mode 100644 index 00000000..8b9396bc --- /dev/null +++ b/src/lightcone/engine/image/errors.py @@ -0,0 +1,70 @@ +"""Error taxonomy for the image layer. + +Every refusal in the container hatch is a distinct exception type with a +pointed, actionable message — a build or declaration problem must never +surface as a downstream mystery (the Modal lesson from the design +record: base flexibility only with a published contract, each violation +a refusal at the earliest possible moment). +""" +from __future__ import annotations + + +class ImageError(Exception): + """Base for every image-layer failure.""" + + +class DeclarationError(ImageError): + """A static ``[tool.lightcone.image]`` / ``Containerfile.extra`` + problem, detectable at parse time: tag-only base, unknown key, bad + apt package name, ``FROM`` inside the extra stage.""" + + +class BaseContractError(ImageError): + """The declared base image violates the base contract (musl-based, + apt-less with system-packages declared, unsupported platform). + Detected at build time — the contract-check layer inside the + generated Containerfile fails with a distinct exit code that the + builder maps back to this error.""" + + +class AptPackageNotFoundError(ImageError): + """apt could not locate a declared system package.""" + + def __init__(self, package: str) -> None: + self.package = package + super().__init__( + f"no apt package named `{package}` — search with " + f"`apt-cache search {package}`" + ) + + +class ImageBuildError(ImageError): + """The image build failed for a reason no more specific error + covers; carries a bounded log tail.""" + + +class PodmanUnavailableError(ImageError): + """podman is not on PATH.""" + + +class MachinePreflightError(ImageError): + """macOS: the podman machine VM is missing, stopped, or does not + share a required mount source.""" + + +class ImageMissingError(ImageError): + """The pinned image is absent from the local store. The message + embeds the exact ``lc build`` command — ``lc run`` never builds.""" + + +def require_podman(podman: str = "podman") -> None: + """One podman-availability gate for builder and runtime alike.""" + import shutil + + if shutil.which(podman) is None: + raise PodmanUnavailableError( + "podman is not on PATH — containerized projects build and " + "execute inside their environment image. Install podman " + "(https://podman.io/docs/installation); `lc status` shows " + "this host's readiness." + ) diff --git a/src/lightcone/engine/image/identity.py b/src/lightcone/engine/image/identity.py new file mode 100644 index 00000000..cd5daa2d --- /dev/null +++ b/src/lightcone/engine/image/identity.py @@ -0,0 +1,56 @@ +"""Content-addressed image identity. + +``tag = lc-env-`` + +The rendered text already embeds every other identity input verbatim — +base digest, uv digest, interpreter version, sorted apt list, extra +stage — so hashing it plus the two build-context files covers the spec +§3 env-input document exactly. ``pyproject.toml`` and ``uv.lock`` are +hashed raw-bytes because they enter the build context and determine +``/opt/venv`` (raw-bytes over-invalidation is the carried-over honest +boundary). Project code contributes nothing (G5): code edits never +move the tag. +""" +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path + +from lightcone.engine.image.errors import DeclarationError +from lightcone.engine.image.render import RenderedContainerfile +from lightcone.engine.manifest import frame + +TAG_PREFIX = "lc-env-" + + +@dataclass(frozen=True) +class EnvInputs: + """The two project files that enter the build context.""" + + pyproject_bytes: bytes + uv_lock_bytes: bytes + + @classmethod + def read(cls, project: Path) -> EnvInputs: + pyproject = project / "pyproject.toml" + uv_lock = project / "uv.lock" + for p in (pyproject, uv_lock): + if not p.is_file(): + raise DeclarationError( + f"{p} is missing — the image is rendered from the " + "locked environment. Run `uv lock` (or `lc init`) first." + ) + return cls( + pyproject_bytes=pyproject.read_bytes(), + uv_lock_bytes=uv_lock.read_bytes(), + ) + + +def compute_tag(rendered: RenderedContainerfile, inputs: EnvInputs) -> str: + h = hashlib.sha256() + frame(h, "containerfile", rendered.text.encode("utf-8")) + frame(h, "pyproject", inputs.pyproject_bytes) + frame(h, "uv.lock", inputs.uv_lock_bytes) + return f"{TAG_PREFIX}{h.hexdigest()[:16]}" diff --git a/src/lightcone/engine/image/machine.py b/src/lightcone/engine/image/machine.py new file mode 100644 index 00000000..a456156c --- /dev/null +++ b/src/lightcone/engine/image/machine.py @@ -0,0 +1,86 @@ +"""macOS ``podman machine`` preflight (spec §5). + +podman on macOS runs containers in a one-time Linux VM. Two failure +modes get refusals with the exact fix — never a mysterious error or, +worse, a silently empty mount: + +* no machine / machine stopped → the one-time setup commands; +* a mount source outside the VM's shared directories → the + ``podman machine set --volume`` command naming the source. + +Linux is a no-op. (Designed here, exercised on macOS CI/manual runs — +this module is deliberately pure-subprocess so the JSON-shape unit +tests cover the logic.) +""" +from __future__ import annotations + +import json +import platform +import subprocess +from pathlib import Path + +from lightcone.engine.image.errors import MachinePreflightError + +#: Default shares podman machine configures when none are listed. +_DEFAULT_SHARES = ("/Users", "/private", "/var/folders") + + +def machine_preflight(sources: list[Path], *, podman: str = "podman") -> None: + if platform.system() != "Darwin": + return + inspect = _machine_inspect(podman) + if inspect is None: + raise MachinePreflightError( + "podman on macOS needs its Linux VM (a one-time setup, " + "~minutes):\n" + " podman machine init\n" + " podman machine start\n" + "note: no GPU is available inside the VM." + ) + if inspect.get("State") != "running": + raise MachinePreflightError( + "the podman machine VM is not running — start it with:\n" + " podman machine start" + ) + shares = _shares(inspect) + for source in sources: + real = source.resolve() + if not any(real.is_relative_to(share) for share in shares): + raise MachinePreflightError( + f"{real} lies outside the podman machine's shared " + "directories — the mount would be silently empty inside " + "the VM. Share it with:\n" + f" podman machine set --volume {real}\n" + " podman machine stop && podman machine start" + ) + + +def _machine_inspect(podman: str) -> dict[str, object] | None: + try: + proc = subprocess.run( + [podman, "machine", "inspect"], + capture_output=True, + text=True, + check=False, + ) + except OSError: + return None + if proc.returncode != 0: + return None + try: + machines = json.loads(proc.stdout) + except json.JSONDecodeError: + return None + return machines[0] if machines else None + + +def _shares(inspect: dict[str, object]) -> tuple[Path, ...]: + mounts = inspect.get("Mounts") + if not isinstance(mounts, list): + mounts = [] + shares = [ + Path(str(m["Source"])) + for m in mounts + if isinstance(m, dict) and m.get("Source") + ] + return tuple(shares) if shares else tuple(Path(s) for s in _DEFAULT_SHARES) diff --git a/src/lightcone/engine/image/mounts.py b/src/lightcone/engine/image/mounts.py new file mode 100644 index 00000000..87078ffe --- /dev/null +++ b/src/lightcone/engine/image/mounts.py @@ -0,0 +1,111 @@ +"""The engine container's mount set — mounts bound the world (spec §7). + +The container sees: the project tree at its **identical absolute path** +(RW for materialize — the engine writes results and manifests; RO for +probes), each declared external input RO, tmpfs ``/tmp`` and +``/dev/shm`` — and nothing else from the host. The environment is baked +(``/opt/venv``); no env mounts exist. +""" +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path + +from lightcone.engine.image.errors import DeclarationError + + +@dataclass(frozen=True) +class MountSet: + project: Path # realpath'd + external_inputs: tuple[Path, ...] # realpath'd, deduped, RO + readonly_project: bool = False # probe posture + + def to_podman_args(self) -> list[str]: + project_mode = "ro" if self.readonly_project else "rw" + args = [ + "-v", f"{self.project}:{self.project}:{project_mode}", + ] + for inp in self.external_inputs: + args += ["-v", f"{inp}:{inp}:ro"] + args += [ + "--tmpfs", "/tmp:rw,exec", + # podman's default /dev/shm is 64MB — a classic scientific- + # workload footgun (multiprocessing, torch DataLoader). + "--shm-size", "1g", + ] + return args + + def sources(self) -> list[Path]: + """Every host path the container mounts — the macOS machine + share-preflight checks each one.""" + return [self.project, *self.external_inputs] + + +def compute_mount_set( + project: Path, + *, + external_inputs: Iterable[Path] = (), + readonly_project: bool = False, +) -> MountSet: + project = project.resolve() + deduped: list[Path] = [] + for raw in external_inputs: + p = Path(raw).resolve() + if not p.exists(): + continue + if p == project or p.is_relative_to(project): + continue # in-tree: the project mount covers it + if project.is_relative_to(p): + raise DeclarationError( + f"declared input {p} is a parent of the project root — " + "mounting it would silently widen the container's world. " + "Declare the specific files/directories instead." + ) + if any(p == d or p.is_relative_to(d) for d in deduped): + continue + deduped = [d for d in deduped if not d.is_relative_to(p)] + deduped.append(p) + return MountSet( + project=project, + external_inputs=tuple(sorted(deduped)), + readonly_project=readonly_project, + ) + + +def external_input_paths( + project: Path, spec: dict[str, object] | None = None +) -> tuple[Path, ...]: + """Union of resolved external input paths across every output's + declared inputs — what the container must mount RO and a probe may + read. Resolution goes through + :func:`lightcone.engine.tree.resolve_external_input`, the single + home for input-source semantics (it follows ``from:`` alias hops a + raw ``source:`` walk would miss). In-tree paths are covered by the + project mount/grant and skipped here. + """ + from astra.helpers import load_yaml, resolve_analysis_tree + + from lightcone.engine.tree import ( + collect_tree_outputs, + find_upstream_output, + resolve_external_input, + ) + + if spec is None: + spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) + tree_outputs = collect_tree_outputs(spec) + paths: list[Path] = [] + for to in tree_outputs: + for inp_id in to.output_def.get("inputs") or []: + if find_upstream_output(to, inp_id, tree_outputs) is not None: + continue # sibling output — mounted with the project + source = resolve_external_input(to, inp_id, spec) + if not source: + continue + p = Path(source) + if not p.is_absolute(): + p = project / p + if p.exists(): + paths.append(p.resolve()) + return tuple(dict.fromkeys(paths)) diff --git a/src/lightcone/engine/image/record.py b/src/lightcone/engine/image/record.py new file mode 100644 index 00000000..64259610 --- /dev/null +++ b/src/lightcone/engine/image/record.py @@ -0,0 +1,66 @@ +"""The build record: what was built, from what, with what result. + +Lives at ``.lightcone/image/record.json`` (machine-local, gitignored — +digests differ per architecture and image store). The dpkg snapshot +*text* is archived beside it so the system-layer attestation outlives +image garbage-collection; manifests store only its sha256. +""" +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from pathlib import Path + +RECORD_DIR = ".lightcone/image" +RECORD_NAME = "record.json" +_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class BuildRecord: + tag: str + image_id: str # config digest — always present locally; the run pin + digest: str | None # manifest digest (registry spelling), if reported + platform: str # e.g. "linux/amd64" + env_version: str + lc_version: str + base: str # digest-pinned base ref + built_at: str # ISO 8601, informational + dpkg_snapshot_sha256: str + + def to_json(self) -> str: + return json.dumps( + {"schema_version": _SCHEMA_VERSION, **asdict(self)}, + indent=2, + sort_keys=True, + ) + + +def record_dir(project: Path) -> Path: + return project / RECORD_DIR + + +def read_record(project: Path) -> BuildRecord | None: + path = record_dir(project) / RECORD_NAME + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + if data.pop("schema_version", None) != _SCHEMA_VERSION: + return None + try: + return BuildRecord(**data) + except TypeError: + return None + + +def write_record(project: Path, record: BuildRecord, snapshot_text: str) -> None: + directory = record_dir(project) + directory.mkdir(parents=True, exist_ok=True) + (directory / f"dpkg-snapshot-{record.tag}.txt").write_text(snapshot_text) + (directory / RECORD_NAME).write_text(record.to_json()) + + +def snapshot_sha256(snapshot_text: str) -> str: + return hashlib.sha256(snapshot_text.encode("utf-8")).hexdigest() diff --git a/src/lightcone/engine/image/render.py b/src/lightcone/engine/image/render.py new file mode 100644 index 00000000..cc96b677 --- /dev/null +++ b/src/lightcone/engine/image/render.py @@ -0,0 +1,145 @@ +"""Deterministic Containerfile rendering. + +``render()`` is a pure function of an :class:`ImageDefinition`: +byte-identical output for equal definitions (fixed layer order, sorted +packages, one whitespace style, LF endings, no timestamps) — the +rendered text is half of the image tag's identity, so any +nondeterminism here would silently fork tags. + +Invariants pinned by golden tests: + +* the offline ENV (``UV_OFFLINE`` …) appears **only in the final + stage** — the build's own ``uv sync`` layer keeps network; +* the apt layer and its contract check exist iff ``system_packages`` + is nonempty; +* the dpkg snapshot is taken in the final stage, *after* the extra + stage, so packages an extra stage installs are attested too. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from lightcone.engine.image import constants +from lightcone.engine.image.definition import ImageDefinition +from lightcone.engine.manifest import canonical_json + + +@dataclass(frozen=True) +class RenderedContainerfile: + text: str + definition: ImageDefinition + + +def render(defn: ImageDefinition) -> RenderedContainerfile: + blocks: list[str] = [] + + blocks.append( + "# generated by lightcone-cli — DO NOT EDIT\n" + "# (rendered from pyproject.toml [tool.lightcone.image]; " + "regenerated by `lc build`)\n" + f"FROM {defn.base} AS env\n" + 'SHELL ["/bin/sh", "-c"]' + ) + + blocks.append(_contract_check_layer(defn)) + + if defn.system_packages: + blocks.append(_apt_layer(defn)) + + blocks.append( + "# pinned uv binary (manifest-list digest — arch-independent text)\n" + f"COPY --from={defn.uv.copy_from_ref} /uv {constants.UV_BIN}" + ) + + blocks.append( + "# exact interpreter, uv-managed, outside the base's control\n" + f"ENV UV_PYTHON_INSTALL_DIR={constants.OPT_PYTHON}\n" + f"RUN {constants.UV_BIN} python install {defn.python_version}" + ) + + blocks.append( + "# locked sync — code-free: pyproject.toml + uv.lock are the only\n" + "# project files that ever enter an image (G5)\n" + f"WORKDIR {constants.PROJECT_STAGE_DIR}\n" + "COPY pyproject.toml uv.lock ./\n" + f"ENV UV_PROJECT_ENVIRONMENT={constants.OPT_VENV}\n" + f"RUN {constants.UV_BIN} sync --locked --exact --no-install-project " + f"--compile-bytecode --python {defn.python_version} " + f"--project {constants.PROJECT_STAGE_DIR}" + ) + + final_from = "env" + if defn.extra_stage is not None: + final_from = "extra" + blocks.append( + "# user extra stage (Containerfile.extra, verbatim)\n" + "FROM env AS extra\n" + defn.extra_stage.strip() + ) + + blocks.append(_final_stage(defn, final_from)) + + text = "\n\n".join(blocks) + "\n" + return RenderedContainerfile(text=text, definition=defn) + + +def _contract_check_layer(defn: ImageDefinition) -> str: + lines = [ + "# base contract checks — each failure is a distinct exit code the", + "# builder maps to a pointed error (never a raw build log)", + f"RUN test -x /bin/sh || exit {constants.EXIT_NO_SH}", + ( + "RUN if ls /lib/ld-musl-* >/dev/null 2>&1; then " + f"echo 'musl base' >&2; exit {constants.EXIT_MUSL_BASE}; fi" + ), + ] + if defn.system_packages: + lines.append( + "RUN command -v apt-get >/dev/null 2>&1 || " + f"exit {constants.EXIT_NO_APT}" + ) + return "\n".join(lines) + + +def _apt_layer(defn: ImageDefinition) -> str: + pkgs = " ".join(defn.system_packages) # already sorted in the declaration + return ( + "# system layer — installed BEFORE uv sync so lock-level system\n" + "# dependencies (sdist builds, rpy2-style imports) resolve here\n" + "RUN apt-get update \\\n" + " && DEBIAN_FRONTEND=noninteractive apt-get install -y " + "--no-install-recommends \\\n" + f" {pkgs} \\\n" + " && rm -rf /var/lib/apt/lists/*" + ) + + +def _final_stage(defn: ImageDefinition, final_from: str) -> str: + identity = canonical_json( + { + "env_version": defn.env_version, + "python_version": defn.python_version, + "uv_version": defn.uv.version, + } + ) + offline_env = " ".join(f"{k}={v}" for k, v in constants.OFFLINE_ENV.items()) + path = ( + f"{constants.OPT_VENV}/bin:{constants.LC_DIR}/bin:" + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ) + return ( + f"FROM {final_from} AS final\n" + f'LABEL io.lightcone.env-version="{defn.env_version}"\n' + "# attestation: name-pinned apt layer's actual package versions\n" + f"RUN dpkg -l > {constants.DPKG_SNAPSHOT_PATH} 2>/dev/null " + f"|| echo 'dpkg unavailable' > {constants.DPKG_SNAPSHOT_PATH}\n" + f"RUN printf '%s' '{identity}' > {constants.IDENTITY_PATH}\n" + "# world-readable: the invoking uid (rootless --userns=keep-id)\n" + "# must be able to read everything lc baked\n" + f"RUN chmod -R a+rX {constants.LC_DIR} {constants.OPT_PYTHON} " + f"{constants.OPT_VENV}\n" + "# offline overlay — FINAL stage only; earlier stages keep network\n" + f"ENV {offline_env} \\\n" + f" UV_PROJECT_ENVIRONMENT={constants.OPT_VENV} \\\n" + f" UV_PYTHON_INSTALL_DIR={constants.OPT_PYTHON} \\\n" + f" PATH={path}" + ) diff --git a/src/lightcone/engine/image/runtime_podman.py b/src/lightcone/engine/image/runtime_podman.py new file mode 100644 index 00000000..5128deb9 --- /dev/null +++ b/src/lightcone/engine/image/runtime_podman.py @@ -0,0 +1,103 @@ +"""Digest-pinned podman execution — the containerized full stack. + +One ``podman run`` hosts everything (spec §1): the delegated engine, +its LocalCluster dask workers, the child snakemake, and every recipe — +all from the baked ``/opt/venv``. The invocation is pinned **by image +id** at the argv (a retagged image cannot substitute), runs under +``--net=none`` (loopback intact — in-container LocalCluster keeps +working) with ``--userns=keep-id`` (the invoking uid owns project +writes) and the entrypoint cleared (base ``ENTRYPOINT``/``CMD``/ +``USER`` are inert, §2). + +Honesty note: a process inside a container cannot independently observe +its own image digest. What IS assertable: (a) the host-side launcher +pins by image id at the argv after checking the store against the build +record — substitution is impossible at the pin point; (b) inside, the +baked ``identity.json`` env_version must equal the job's expected +env_version — a *content* assertion independent of the wrapper; (c) +``LC_IMAGE_DIGEST`` equals the job-command digest — on the +single-container laptop path this is self-consistent rather than +independent. The manifest records mechanism honestly, never claiming +(c) is stronger than it is. +""" +from __future__ import annotations + +import os +import sys +from collections.abc import Sequence +from typing import NoReturn + +from lightcone.engine.contract import ( + CONTAINER_NETWORK_ENV, + CONTAINER_RUNTIME_VALUE, + DELEGATED_ENV, + IMAGE_DIGEST_ENV, + WORKER_RUNTIME_ENV, +) +from lightcone.engine.image import constants +from lightcone.engine.image.errors import require_podman +from lightcone.engine.image.mounts import MountSet +from lightcone.engine.image.record import BuildRecord + +#: Ambient variables passed through into the container — a closed +#: allowlist, never the ambient environment wholesale. +_ENV_PASSTHROUGH = ("TERM", "COLUMNS", "LINES", "LANG") + + +class PodmanRuntime: + def __init__(self, podman: str = "podman") -> None: + require_podman(podman) + self._podman = podman + + def run_argv( + self, + *, + record: BuildRecord, + mounts: MountSet, + argv: Sequence[str], + interactive: bool = False, + ) -> list[str]: + cmd = [ + self._podman, "run", "--rm", "--pull=never", + # --net=none denies egress with loopback intact — what the + # hermeticity record's `network: denied` means. + "--net=none", + "--userns=keep-id", + "--entrypoint=", + # SELinux hosts: never relabel user data. + "--security-opt", "label=disable", + ] + if interactive: + cmd.append("-it") + cmd += mounts.to_podman_args() + cmd += ["-w", str(mounts.project)] + cmd += [ + "-e", f"{DELEGATED_ENV}=1", + "-e", f"{WORKER_RUNTIME_ENV}={CONTAINER_RUNTIME_VALUE}", + "-e", f"{CONTAINER_NETWORK_ENV}=none", + "-e", f"{IMAGE_DIGEST_ENV}={record.image_id}", + ] + for name in _ENV_PASSTHROUGH: + if name in os.environ: + cmd += ["-e", f"{name}={os.environ[name]}"] + # The pin point: run BY image id. + cmd.append(record.image_id) + cmd.extend(argv) + return cmd + + def exec_full_stack( + self, + *, + record: BuildRecord, + mounts: MountSet, + lc_argv: Sequence[str], + ) -> NoReturn: + """Direct exec (§4 step 5): re-enter ``lc`` from the image's + baked env. Never returns.""" + argv = self.run_argv( + record=record, + mounts=mounts, + argv=[f"{constants.OPT_VENV}/bin/lc", *lc_argv], + interactive=sys.stdin.isatty(), + ) + os.execvp(self._podman, argv) diff --git a/src/lightcone/engine/job.py b/src/lightcone/engine/job.py new file mode 100644 index 00000000..28d4895c --- /dev/null +++ b/src/lightcone/engine/job.py @@ -0,0 +1,42 @@ +"""The typed per-(rule, universe) job contract. + +One value object defines what travels from the Snakefile generator +through ``snakefile-config.json`` into the worker's ``run_rule`` — +replacing ad-hoc dict keys with a schema both sides share. The JSON +form (``to_cfg``/``from_cfg``) is what ``params.cfg`` carries and what +``write_manifest`` consumes. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass, field, fields +from typing import Any + + +@dataclass(frozen=True) +class RuleJob: + output_id: str + universe_id: str + recipe: str # the raw authored template (recorded in the manifest) + shell_command: str # rendered, code_version-prefixed + code_version: str + env_version: str # the mid-run gates' baseline + output_type: str | None = None + decisions: dict[str, Any] = field(default_factory=dict) + writable_project: bool = False + sdist_built: list[str] = field(default_factory=list) + git_sha: str | None = None + git_dirty: bool | None = None + git_remote: str | None = None + lc_version: str | None = None + worker_runtime: str = "host" # "host" | "container" + image_tag: str | None = None + image_digest: str | None = None # driver-resolved; asserted worker-side + dpkg_snapshot_sha256: str | None = None + + def to_cfg(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_cfg(cls, cfg: dict[str, Any]) -> RuleJob: + known = {f.name for f in fields(cls)} + return cls(**{k: v for k, v in cfg.items() if k in known}) diff --git a/src/lightcone/engine/manifest.py b/src/lightcone/engine/manifest.py index b22fda1b..1eda7e2d 100644 --- a/src/lightcone/engine/manifest.py +++ b/src/lightcone/engine/manifest.py @@ -4,21 +4,34 @@ sidecar JSON manifest at ``/.lightcone-manifest.json`` that records: -- ``code_version``: sha256(recipe + container image + decisions). Stored - in each rule's per-universe ``params.cfg`` so Snakemake's ``params`` - rerun-trigger detects drift automatically. (The ``code`` trigger only - sees the rule body source, which is universe-parameterized and never - changes — that is why ``lc run`` defaults to including ``params``.) +- ``code_version``: sha256(recipe + decisions + env_version [+ the + output's writable-project sandbox escalation]). Stored in each rule's + per-universe ``params.cfg`` so Snakemake's ``params`` rerun-trigger + detects drift automatically. (The ``code`` trigger only sees the rule + body source, which is universe-parameterized and never changes — that + is why ``lc materialize`` defaults to including ``params``.) +- ``env_version``: the environment identity (lock + interpreter pin + + install settings + system layer) — see + :mod:`lightcone.engine.environment`. - ``data_version``: sha256 of the output directory's contents. Lets ``lc verify`` prove the bytes on disk are what the manifest claims. - ``input_versions``: each declared input's ``data_version`` (if it's a materialized output) or ``(mtime, size)`` fingerprint (if it's an external file). This is the chain. +- ``hermeticity``: what enforcement the recipe actually ran under + (mechanism, file scope, network posture) — recorded from the applied + flags, never from documentation. +- runtime attestation: platform, interpreter build, uv version, env + snapshot, GPU driver — captured worker-side, inside the boundary. Manifests are written by :func:`write_manifest`, called from each rule's -``run:`` block on the host immediately after the recipe shell exits. The -``os.replace`` rename is the atomic commit point: either the rule produced -both data and manifest, or it failed and Snakemake will rerun it. +``run:`` block immediately after the recipe exits. The ``os.replace`` +rename is the atomic commit point: either the rule produced both data +and manifest, or it failed and Snakemake will rerun it. + +Manifests from earlier schema versions read as *pre-migration* +(:func:`is_pre_migration`): status shows them distinctly, verify still +checks the hashes they carry. """ from __future__ import annotations @@ -31,18 +44,29 @@ from typing import Any MANIFEST_FILENAME = ".lightcone-manifest.json" -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 #: Filenames inside an output directory that the data_version hash MUST #: ignore: the manifest itself (chicken-and-egg) and Snakemake's #: ``directory()`` mtime marker (touched AFTER the rule body completes). _HASH_EXCLUDE = frozenset({MANIFEST_FILENAME, ".snakemake_timestamp"}) +#: The honest record for an exec that ran with no enforcement mechanism. +UNSANDBOXED_HERMETICITY = { + "mechanism": "none", + "fs": "open", + "network": "allowed", +} + __all__ = [ "MANIFEST_FILENAME", "SCHEMA_VERSION", + "UNSANDBOXED_HERMETICITY", + "canonical_json", "code_version", "fingerprint_external", + "frame", + "is_pre_migration", "read_manifest", "sha256_dir", "write_manifest", @@ -84,6 +108,23 @@ def _sha256_bytes(data: bytes) -> str: return f"sha256:{hashlib.sha256(data).hexdigest()}" +def canonical_json(obj: Any) -> str: + """The one canonical JSON form for anything that feeds an identity + hash — a formatting drift here forks every hash downstream.""" + return json.dumps(obj, sort_keys=True, separators=(",", ":")) + + +def frame(h: hashlib._Hash, label: str, data: bytes) -> None: + """Length-framed hash update — prevents boundary-shifting collisions + between concatenated identity inputs. Shared by ``env_version`` and + the image tag.""" + h.update(label.encode("utf-8")) + h.update(b"\0") + h.update(str(len(data)).encode("ascii")) + h.update(b"\0") + h.update(data) + + def _sha256_file(path: Path) -> str: h = hashlib.sha256() _hash_file(path, h) @@ -110,25 +151,31 @@ def fingerprint_external(path: Path, *, strict: bool = False) -> str: def code_version( *, recipe: str, - container_image: str | None, decisions: dict[str, Any], + env_version: str, + writable_project: bool = False, ) -> str: """Compute a deterministic code version for an output. - Hashes the recipe text, container image identifier, and canonicalized - decisions. Anything that changes the materialization semantics flows - through this hash; the *runtime* used to invoke the container - (docker/podman/podman-hpc) is intentionally excluded — the same image - produces the same data regardless of which OCI tool launched it. + Hashes the recipe text, canonicalized decisions, the environment + identity, and the output's sandbox escalation (``writable-project`` + widens what a recipe can read back from its own earlier writes, so + it is materialization-relevant — but per-output, so escalating one + output never stales its siblings). Anything that changes the + materialization semantics flows through this hash. """ payload = { "recipe": recipe, - "container_image": container_image, "decisions": decisions, + "env_version": env_version, + "writable_project": writable_project, } - return _sha256_bytes( - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") - ) + return _sha256_bytes(canonical_json(payload).encode("utf-8")) + + +def is_pre_migration(manifest: dict[str, Any]) -> bool: + """True when *manifest* predates the v2 schema (no ``env_version``).""" + return manifest.get("schema_version") != SCHEMA_VERSION or "env_version" not in manifest def read_manifest(output_dir: Path) -> dict[str, Any] | None: @@ -153,10 +200,12 @@ def write_manifest( output_dir: Path, inputs: dict[str, Path], cfg: dict[str, Any], + hermeticity: dict[str, Any] | None = None, + attestation: dict[str, Any] | None = None, ) -> Path: """Atomically write the manifest for an already-materialized output. - Called from each rule's ``run:`` block after the recipe shell exits. + Called from each rule's ``run:`` block after the recipe exits. Hashes the output dir, resolves input versions (chaining to upstream manifests when present, falling back to external fingerprints), and commits the manifest via ``os.replace``. @@ -167,8 +216,18 @@ def write_manifest( either a directory containing a sibling manifest (upstream output) or an external file/dir. cfg: Per-rule configuration. Required keys: ``output_id``, - ``universe_id``, ``recipe``, ``container_image``, ``decisions``, - ``code_version``, ``git_sha``, ``lc_version``. + ``universe_id``, ``recipe``, ``decisions``, ``code_version``, + ``env_version``. Optional: ``git_sha``, ``git_dirty``, + ``git_remote``, ``lc_version``, ``worker_runtime``, + ``image_tag``, ``image_digest``, ``dpkg_snapshot_sha256``, + ``sdist_built``. + hermeticity: The enforcement record for the exec that produced + the data — the *applied* flags, never the documented matrix + row. Defaults to the honest unsandboxed record. + attestation: Runtime-environment capture from + :func:`lightcone.engine.attestation.capture_runtime_attestation`, + taken worker-side (hence inside the boundary in containerized + mode). """ output_dir = Path(output_dir) @@ -181,32 +240,34 @@ def write_manifest( else: input_versions[inp_id] = fingerprint_external(inp_path) + image = ( + {"tag": cfg["image_tag"], "digest": cfg.get("image_digest")} + if cfg.get("image_tag") + else None + ) + manifest = { "schema_version": SCHEMA_VERSION, "output_id": cfg["output_id"], "universe_id": cfg["universe_id"], "code_version": cfg["code_version"], "data_version": sha256_dir(output_dir), - "container_image": cfg.get("container_image"), + "env_version": cfg["env_version"], "recipe": cfg["recipe"], "decisions": cfg.get("decisions", {}), "input_versions": input_versions, "git_sha": cfg.get("git_sha"), - # URL of the git origin remote at the time of materialization. - # Optional/additive — older manifests without this field still - # parse. Surfaced by ``lc export wrroc`` as a CodeRepository entity. + "git_dirty": cfg.get("git_dirty"), "git_remote": cfg.get("git_remote"), "lc_version": cfg.get("lc_version"), "finished_at": time.time(), "host": socket.gethostname(), - "slurm_job_id": os.environ.get("SLURM_JOB_ID"), - # On a Dask Gateway deployment `lc run` provisions this into - # every scheduler/worker pod (via the `environment` cluster - # option) with the image the cluster was started with. - # ``container_image`` above is what the spec *declared*; this is - # the pod-reported ground truth of what actually executed. - # Optional/additive — None everywhere else. - "worker_image": os.environ.get("LIGHTCONE_WORKER_IMAGE"), + "worker_runtime": cfg.get("worker_runtime", "host"), + "image": image, + "dpkg_snapshot_sha256": cfg.get("dpkg_snapshot_sha256"), + "sdist_built": cfg.get("sdist_built", []), + "hermeticity": hermeticity or dict(UNSANDBOXED_HERMETICITY), + **(attestation or {}), } final_path = output_dir / MANIFEST_FILENAME diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py new file mode 100644 index 00000000..574fc570 --- /dev/null +++ b/src/lightcone/engine/project.py @@ -0,0 +1,21 @@ +"""Project discovery. + +One rule, shared by the launcher and every CLI verb: the project root +is the nearest ancestor (including the start directory) containing an +``astra.yaml`` file. uv's native walk-up discovery is never trusted — +every uv invocation downstream carries an explicit ``--project ``. +""" +from __future__ import annotations + +from pathlib import Path + +SPEC_FILENAME = "astra.yaml" + + +def find_root(start: Path | None = None) -> Path | None: + """Walk up from *start* (default: cwd) to the project root, or ``None``.""" + p = (start or Path.cwd()).resolve() + for parent in [p, *p.parents]: + if (parent / SPEC_FILENAME).is_file(): + return parent + return None diff --git a/src/lightcone/engine/runner.py b/src/lightcone/engine/runner.py index 800f302a..9e31fd5f 100644 --- a/src/lightcone/engine/runner.py +++ b/src/lightcone/engine/runner.py @@ -1,34 +1,45 @@ """Per-rule execution helper invoked from the generated Snakefile. -Each rule's ``run:`` block boils down to one call to :func:`run_rule`. -The helper: - -* runs the rule's pre-rendered shell command (template substitution and - container wrapping happen at Snakefile-generation time — see - :func:`lightcone.engine.snakefile.render_recipe`) with stdout and - stderr captured, -* emits a ``▶ rule [universe]`` header, the recipe's output, and a - ``✓ rule [universe] `` (or ``✗ … exit=N``) trailer, - each line framed with a sentinel prefix the executor extracts, -* writes the per-output manifest on success, -* runs the validation hook on the materialized output, -* raises :class:`subprocess.CalledProcessError` on non-zero exit so - Snakemake records the job as failed and halts the DAG. - -The sentinel prefix (:data:`SENTINEL`) is what the dask executor's -``_run_shell`` looks for when it filters worker subprocess output — -anything else (snakemake bootstrap, dask logs, stray prints) is dropped -on the floor. This is the entire mechanism by which lc run shows clean, -narrative output without ever filtering against a moving target of -upstream log strings. +Each rule's ``run:`` block boils down to one call to :func:`run_rule`, +which implements the worker sequence (spec §6): + +1. **pre-gate** — recompute ``env_version`` from the project tree and + compare against the value baked into the job at generation time; a + mid-run relock aborts loudly instead of materializing under a + different environment than the manifest will claim. +2. **env check** — direct mode: the env prefix exists and + ``uv sync --locked --exact --check`` passes (a true no-write + env-vs-lock verification); containerized mode: the baked image + identity and the driver-resolved digest match the job's pins. +3. **boundary exec** — the recipe runs through the exec boundary + (:mod:`lightcone.engine.boundary`) with the offline overlay + (converge once, then never write to the environment) and the + ambient ``UV_*`` scrub; ``--require-sandbox`` is enforced here, + worker-side, against the *probed* enforcement level. +4. **post-gate** — ``env_version`` re-checked before + :func:`~lightcone.engine.manifest.write_manifest` commits, so the + double gate brackets the recipe. + +Output is emitted as sentinel-prefixed lines (:data:`SENTINEL`) the +dask executor extracts; on non-zero exit the manifest is **not** +written and :class:`subprocess.CalledProcessError` propagates so +Snakemake records the job as failed. """ from __future__ import annotations +import json +import os import subprocess import sys import time from pathlib import Path -from typing import Any +from typing import Any, Literal + +from lightcone.engine.contract import ( + IMAGE_DIGEST_ENV, + NO_SANDBOX_ENV, + REQUIRE_SANDBOX_ENV, +) #: Lines from the runner are prefixed with this so ``_run_shell`` in the #: dask executor can distinguish them from snakemake/dask noise. Chosen @@ -37,6 +48,7 @@ SENTINEL = "__LCSTREAM__::" + def _emit(line: str = "") -> None: """Write one sentinel-prefixed line to stdout and flush. @@ -49,6 +61,85 @@ def _emit(line: str = "") -> None: sys.stdout.flush() +class RuleGateError(RuntimeError): + """A worker-side integrity gate refused to run (or commit) the rule.""" + + +def _current_env_version(root: Path) -> str: + from lightcone.engine.environment import load_environment + + return load_environment(root).env_version + + +def _gate_env(root: Path, expected: str, *, when: str) -> None: + actual = _current_env_version(root) + if actual != expected: + raise RuleGateError( + f"environment changed mid-run ({when}): the lock/pyproject " + "no longer matches the environment this run started with — " + "re-run `lc materialize`." + ) + + +def _env_check(root: Path, job: Any) -> None: + """Step 2: verify the execution environment matches the job's pins.""" + if job.worker_runtime == "container": + from lightcone.engine.image.constants import IDENTITY_PATH + + try: + baked = json.loads(Path(IDENTITY_PATH).read_text()) + except (OSError, json.JSONDecodeError) as e: + raise RuleGateError( + f"containerized job outside an lc image ({IDENTITY_PATH} " + f"unreadable: {e})" + ) from e + if baked.get("env_version") != job.env_version: + raise RuleGateError( + "image/environment mismatch: the running image was baked " + f"for env_version {baked.get('env_version')}, the job " + f"expects {job.env_version} — run `lc build`." + ) + running = os.environ.get(IMAGE_DIGEST_ENV) + if job.image_digest and running != job.image_digest: + raise RuleGateError( + f"image digest mismatch: driver pinned {job.image_digest}, " + f"running container reports {running!r}." + ) + return + + venv = root / ".venv" + if not venv.is_dir(): + raise RuleGateError( + f"{venv} does not exist — the environment was never converged." + ) + proc = subprocess.run( + [ + "uv", "sync", "--locked", "--exact", "--check", + "--project", str(root), + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + raise RuleGateError( + "the environment does not match the lock " + f"(`uv sync --check` failed):\n{proc.stderr.strip()}" + ) + + +def _exec_env() -> dict[str, str]: + """The recipe's process environment: ambient minus the UV_* steering + surface, plus the offline overlay — converge once, then never write + to the environment.""" + from lightcone.engine import uv_env + + env = dict(os.environ) + uv_env.scrub(env) + env.update(uv_env.OFFLINE_OVERLAY) + return env + + def run_rule( *, rule_key: str, @@ -57,48 +148,95 @@ def run_rule( inputs: dict[str, Path], cfg: dict[str, Any], ) -> None: - """Execute one rule's pre-rendered shell command and write its manifest. + """Execute one rule through the worker sequence; write its manifest. - Called from the generated Snakefile's ``run:`` block. Recipe stdout - and stderr are interleaved by capture order (stdout first, then - stderr) — Snakemake's own output capture has the same property and - most recipes are well-behaved enough that this is fine. + Called from the generated Snakefile's ``run:`` block with + ``cwd == project root`` (snakemake ``-d``). Recipe stdout and + stderr are interleaved by capture order. On non-zero exit, the manifest is **not** written. Snakemake will treat the rule as failed; ``lc verify`` won't see a stale manifest pointing at incomplete data. """ + from lightcone.engine.attestation import capture_runtime_attestation + from lightcone.engine.boundary import ExecScope, get_boundary + from lightcone.engine.job import RuleJob from lightcone.engine.manifest import write_manifest from lightcone.engine.validation import validate_output + job = RuleJob.from_cfg(cfg) + root = Path.cwd() + t0 = time.monotonic() _emit(f"\033[2m▶\033[0m {rule_key} \033[2m[{universe}]\033[0m") - proc = subprocess.run( - cfg["shell_command"], - shell=True, - capture_output=True, - text=True, - check=False, - ) + try: + _gate_env(root, job.env_version, when="pre-recipe") + _env_check(root, job) + except RuleGateError as e: + _emit(f" \033[31m{e}\033[0m") + raise - for line in proc.stdout.splitlines(): + sandbox_mode: Literal["on", "off"] = ( + "off" if os.environ.get(NO_SANDBOX_ENV) == "1" else "on" + ) + scope = ExecScope( + project_root=root, + output_dir=Path(output_dir), + read_paths=tuple(Path(p) for p in inputs.values()), + writable_project=job.writable_project, + sandbox=sandbox_mode, + ) + boundary = get_boundary() + + # --require-sandbox is enforced worker-side against the probed + # enforcement level — the driver's kernel is not the worker's. + if requirement := os.environ.get(REQUIRE_SANDBOX_ENV): + probed = boundary.probe(scope) + if probed.mechanism == "none": + raise RuleGateError( + "--require-sandbox: no sandbox mechanism is available on " + f"this worker (probed: {probed.mechanism})." + ) + if requirement == "declared-fs" and probed.fs != "declared": + raise RuleGateError( + "--require-sandbox=declared-fs: this worker can only " + f"provide fs: {probed.fs}." + ) + + result = boundary.execute(job.shell_command, scope, env=_exec_env()) + + for line in result.stdout.splitlines(): _emit(f" {line}") - for line in proc.stderr.splitlines(): + for line in result.stderr.splitlines(): _emit(f" {line}") + for note in result.notes: + _emit(f" {note}") dt = time.monotonic() - t0 - if proc.returncode != 0: + if result.returncode != 0: _emit( f"\033[31m✗\033[0m {rule_key} \033[2m[{universe}]\033[0m " - f"exit={proc.returncode} {dt:.1f}s" + f"exit={result.returncode} {dt:.1f}s" ) - raise subprocess.CalledProcessError(proc.returncode, cfg["shell_command"]) - - write_manifest(output_dir=output_dir, inputs=inputs, cfg=cfg) + raise subprocess.CalledProcessError(result.returncode, job.shell_command) + + try: + _gate_env(root, job.env_version, when="post-recipe") + except RuleGateError as e: + _emit(f" \033[31m{e}\033[0m") + raise + + write_manifest( + output_dir=output_dir, + inputs=inputs, + cfg=job.to_cfg(), + hermeticity=result.attestation.to_manifest(), + attestation=capture_runtime_attestation(), + ) for warning in validate_output( - output_dir, cfg.get("output_type"), cfg["output_id"] + output_dir, job.output_type, job.output_id ): _emit(f" \033[33m⚠\033[0m {warning}") @@ -107,4 +245,11 @@ def run_rule( ) -__all__ = ["SENTINEL", "run_rule"] +__all__ = [ + "IMAGE_DIGEST_ENV", + "NO_SANDBOX_ENV", + "REQUIRE_SANDBOX_ENV", + "SENTINEL", + "RuleGateError", + "run_rule", +] diff --git a/src/lightcone/engine/sandbox/__init__.py b/src/lightcone/engine/sandbox/__init__.py new file mode 100644 index 00000000..8fda91de --- /dev/null +++ b/src/lightcone/engine/sandbox/__init__.py @@ -0,0 +1,13 @@ +"""The sandbox/hermeticity layer (spec §7). + +The boundary seam consumes exactly one name from this package: +:class:`~lightcone.engine.sandbox.exec_boundary.SandboxExecBoundary`, +the enforced :class:`~lightcone.engine.boundary.ExecBoundary`. +Everything else (policy, probe, wrap, denial) is addressed by its +submodule. +""" +from __future__ import annotations + +from lightcone.engine.sandbox.exec_boundary import SandboxExecBoundary + +__all__ = ["SandboxExecBoundary"] diff --git a/src/lightcone/engine/sandbox/_landlock.py b/src/lightcone/engine/sandbox/_landlock.py new file mode 100644 index 00000000..ff01a3f3 --- /dev/null +++ b/src/lightcone/engine/sandbox/_landlock.py @@ -0,0 +1,163 @@ +"""Vendored Landlock ctypes bindings — parent side. + +No external dependencies: three raw syscalls and the access-right +constants from ``linux/landlock.h``. The *parent* builds the ruleset FD +(:func:`build_ruleset_fd`); the child-side restrict step lives in +:mod:`lightcone._sandbox_exec`, which deliberately duplicates the two +constants it needs (shim-constant parity is pinned by a unit test). + +Everything here is unprivileged — Landlock needs no capabilities, which +is what makes the enforcement tests runnable in any CI. +""" +from __future__ import annotations + +import ctypes +import ctypes.util +import errno +import functools +import os +import platform +from pathlib import Path + +# asm-generic syscall numbers — identical on x86_64 and aarch64. +SYS_LANDLOCK_CREATE_RULESET = 444 +SYS_LANDLOCK_ADD_RULE = 445 +SYS_LANDLOCK_RESTRICT_SELF = 446 + +LANDLOCK_CREATE_RULESET_VERSION = 1 << 0 +_RULE_PATH_BENEATH = 1 + +# Access-right bits by the ABI that introduced them. +ACCESS_FS_EXECUTE = 1 << 0 +ACCESS_FS_WRITE_FILE = 1 << 1 +ACCESS_FS_READ_FILE = 1 << 2 +ACCESS_FS_READ_DIR = 1 << 3 +ACCESS_FS_REMOVE_DIR = 1 << 4 +ACCESS_FS_REMOVE_FILE = 1 << 5 +ACCESS_FS_MAKE_CHAR = 1 << 6 +ACCESS_FS_MAKE_DIR = 1 << 7 +ACCESS_FS_MAKE_REG = 1 << 8 +ACCESS_FS_MAKE_SOCK = 1 << 9 +ACCESS_FS_MAKE_FIFO = 1 << 10 +ACCESS_FS_MAKE_BLOCK = 1 << 11 +ACCESS_FS_MAKE_SYM = 1 << 12 +_ABI1_ALL = (1 << 13) - 1 +ACCESS_FS_REFER = 1 << 13 # ABI ≥ 2 +ACCESS_FS_TRUNCATE = 1 << 14 # ABI ≥ 3 +ACCESS_FS_IOCTL_DEV = 1 << 15 # ABI ≥ 5 + +READ_BITS = ACCESS_FS_READ_FILE | ACCESS_FS_READ_DIR +#: Rights the kernel accepts on a rule whose parent is a regular file. +_FILE_ONLY_BITS = ( + ACCESS_FS_EXECUTE + | ACCESS_FS_WRITE_FILE + | ACCESS_FS_READ_FILE + | ACCESS_FS_TRUNCATE + | ACCESS_FS_IOCTL_DEV +) + +_SUPPORTED_ARCHES = {"x86_64", "aarch64", "arm64"} + + +class _RulesetAttr(ctypes.Structure): + _fields_ = [ + ("handled_access_fs", ctypes.c_uint64), + ("handled_access_net", ctypes.c_uint64), + ] + + +class _PathBeneathAttr(ctypes.Structure): + # The kernel struct is packed (u64 + s32). + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("allowed_access", ctypes.c_uint64), + ("parent_fd", ctypes.c_int32), + ] + + +@functools.cache +def _libc() -> ctypes.CDLL: + return ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + + +@functools.cache +def abi() -> int: + """The kernel's Landlock ABI level; 0 when unavailable. + + Doubles as the capability probe: a blocked syscall (old kernel, or + a container seccomp profile that filters ``landlock_*``) reads as + unavailable — never as an error. + """ + if platform.machine() not in _SUPPORTED_ARCHES: + return 0 + r = _libc().syscall( + SYS_LANDLOCK_CREATE_RULESET, None, 0, LANDLOCK_CREATE_RULESET_VERSION + ) + return int(r) if r > 0 else 0 + + +def handled_access_for(abi_level: int) -> int: + """Every fs bit this ABI can handle — never pass unknown bits + (EINVAL); anything handled-but-not-granted is denied, which is the + allowlist semantics.""" + bits = _ABI1_ALL + if abi_level >= 2: + bits |= ACCESS_FS_REFER + if abi_level >= 3: + bits |= ACCESS_FS_TRUNCATE + if abi_level >= 5: + bits |= ACCESS_FS_IOCTL_DEV + return bits + + +def write_bits(abi_level: int) -> int: + """The full write group for this ABI (a writable subtree gets + everything: create/remove/link plus read-back). + + ABI-1 caveat, documented: without REFER, cross-directory rename or + link into the write set is denied wholesale — the denial UX + recognizes the resulting EXDEV. + """ + bits = _ABI1_ALL # includes all MAKE_*/REMOVE_* + read + write + exec? no exec + bits &= ~ACCESS_FS_EXECUTE + if abi_level >= 2: + bits |= ACCESS_FS_REFER + if abi_level >= 3: + bits |= ACCESS_FS_TRUNCATE + return bits + + +def create_ruleset(handled_fs: int) -> int: + attr = _RulesetAttr(handled_fs, 0) + fd = _libc().syscall( + SYS_LANDLOCK_CREATE_RULESET, ctypes.byref(attr), ctypes.sizeof(attr), 0 + ) + if fd < 0: + raise OSError(ctypes.get_errno(), "landlock_create_ruleset failed") + return int(fd) + + +def add_path_rule(ruleset_fd: int, path: Path, access: int) -> None: + """Grant *access* beneath *path*. Missing paths raise FileNotFoundError + (callers decide skip-vs-fail); file paths get dir-only bits masked.""" + parent_fd = os.open(path, os.O_PATH | os.O_CLOEXEC) + try: + if not os.fstat(parent_fd).st_mode & 0o040000: # not a directory + access &= _FILE_ONLY_BITS + if not access: + return + attr = _PathBeneathAttr(access, parent_fd) + r = _libc().syscall( + SYS_LANDLOCK_ADD_RULE, + ruleset_fd, + _RULE_PATH_BENEATH, + ctypes.byref(attr), + 0, + ) + if r != 0: + e = ctypes.get_errno() + raise OSError(e, f"landlock_add_rule({path}): {errno.errorcode.get(e, e)}") + finally: + os.close(parent_fd) + diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py new file mode 100644 index 00000000..00bbbfd6 --- /dev/null +++ b/src/lightcone/engine/sandbox/denial.py @@ -0,0 +1,171 @@ +"""The denial UX — the design's primary UI (spec §7, mandatory). + +When a sandboxed recipe fails, the *unsandboxed parent* re-stats the +paths named in the error output and classifies each confirmed denial as +a **tool** (executable / bin-dir path) or a **data file**, then renders +the two-remedy message: the copy-pasteable ``[tool.lightcone.image]`` +fix (with its cost stated) and the ``astra.yaml`` input fix — ordering +by the classification, both always shown. Escape hatches live in a +subdued diagnostics trailer, never as peer remedies. + +And on **every** nonzero sandboxed exit — including recipes that +swallow the PermissionError, and rewrapped errors that defeat the +re-stat classifier — a fixed one-line trailer points at +``lc run --sandbox-debug``, so a denial can never fail invisibly. + +Pure functions over captured output: rendered worker-side and emitted +through the SENTINEL stream; the same renderer serves ``lc run`` +driver-side. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +from lightcone.engine.sandbox.hints import apt_hint +from lightcone.engine.sandbox.model import SandboxPolicy + +#: Path-bearing error shapes recipes commonly surface. +_CANDIDATE_RES = ( + # Python: PermissionError: [Errno 13] Permission denied: '/path' + re.compile(r"(?:PermissionError|FileNotFoundError).*?['\"]([^'\"]+)['\"]"), + # bash: line 1: /path: Permission denied + re.compile(r"(?:bash|sh): (?:line \d+: )?([^\s:]+): Permission denied"), + # bash: cmd: command not found + re.compile(r"(?:bash|sh): (?:line \d+: )?([^\s:]+): command not found"), + # OSError: [Errno 18] Invalid cross-device link (Landlock ABI-1 REFER) + re.compile(r"Invalid cross-device link.*?['\"]([^'\"]+)['\"]"), +) + +_BIN_DIR_HINTS = ("/bin", "/sbin", "/Library/TeX", "/opt") + + +def _in_policy(path: Path, policy: SandboxPolicy, *, kind: str) -> bool: + """Access-aware membership: an executable being in the READ baseline + (e.g. under /usr) does not make its *execution* granted — check the + set matching the classified access.""" + resolved = Path(os.path.realpath(path)) + granted_sets = ( + (policy.execute,) if kind == "tool" else (policy.read, policy.write) + ) + for granted_set in granted_sets: + for granted in granted_set: + try: + resolved.relative_to(granted) + return True + except ValueError: + continue + return False + + +def _classify(path: Path) -> str: + """'tool' or 'data' — ordering heuristic only; both remedies always + render.""" + if os.access(path, os.X_OK) and path.is_file(): + return "tool" + if any(str(path.parent).endswith(h) or h in str(path) for h in _BIN_DIR_HINTS): + return "tool" + return "data" + + +def explain_failure( + *, + stdout: str, + stderr: str, + policy: SandboxPolicy, +) -> list[str]: + """Render the denial message; empty when no denial is confirmed. + + The re-stat step is what separates a sandbox denial from an + ordinary recipe bug: a path that exists on the host but lies + outside the policy sets is a confirmed denial; a path that truly + does not exist is an ordinary error (the trailer still fires). + """ + combined = stdout + "\n" + stderr + candidates: list[str] = [] + for regex in _CANDIDATE_RES: + candidates.extend(regex.findall(combined)) + + confirmed: list[tuple[Path, str]] = [] + seen: set[str] = set() + for raw in candidates: + if raw in seen: + continue + seen.add(raw) + path = Path(raw) + if not path.is_absolute(): + resolved = _which_on_host(raw) + if resolved is None: + continue + path = resolved + if not path.exists(): + continue + kind = _classify(path) + if not _in_policy(path, policy, kind=kind): + confirmed.append((path, kind)) + + if not confirmed: + return [] + + path, kind = confirmed[0] + tool_name = path.name + hint = apt_hint(tool_name) + pkg = hint or f"" + + tool_remedy = [ + " if this is a tool the recipe needs, declare it in the system layer:", + " [tool.lightcone.image]", + f' system-packages = ["{pkg}"]', + ] + if hint is None: + tool_remedy.append( + f" (apt package names — unsure? try: apt-cache search {tool_name})" + ) + tool_remedy += [ + " note: this containerizes the project — podman required (macOS:", + " one-time `podman machine` VM setup, ~minutes) — and re-stages", + " all materialized outputs.", + ] + + data_remedy = [ + " if this is a data file, declare it as an input in astra.yaml:", + " outputs:", + " :", + " inputs:", + f" - path: {path}", + ] + + remedies = ( + tool_remedy + [""] + data_remedy + if kind == "tool" + else data_remedy + [""] + tool_remedy + ) + verb = "execute" if kind == "tool" else "read" + return [ + f"blocked by lc sandbox: cannot {verb} {path} —", + "not part of the declared environment.", + "", + *remedies, + "", + " diagnostics: lc run --sandbox-debug (shell inside the sandbox) ·", + " lc run --no-sandbox (recorded as unsandboxed) · lc status", + ] + + +def _which_on_host(name: str) -> Path | None: + import shutil + + hit = shutil.which(name) + return Path(hit) if hit else None + + +def trailer(mechanism: str) -> str: + """The fixed line appended to EVERY nonzero sandboxed exit — catches + recipes that swallow the PermissionError and errors that defeat the + classifier.""" + return ( + f"this recipe ran under the lc sandbox ({mechanism}) — if the " + "failure looks like a permissions/missing-file error, try " + "`lc run --sandbox-debug`" + ) diff --git a/src/lightcone/engine/sandbox/exec_boundary.py b/src/lightcone/engine/sandbox/exec_boundary.py new file mode 100644 index 00000000..963bedd3 --- /dev/null +++ b/src/lightcone/engine/sandbox/exec_boundary.py @@ -0,0 +1,136 @@ +"""The sandbox-backed :class:`~lightcone.engine.boundary.ExecBoundary`. + +One boundary for every venue: the capability probe decides the +mechanism (Landlock on Linux — including inside a podman container, +where the same code path answers the seccomp question; Seatbelt on +macOS; none elsewhere), the policy realizes spec §7's declared sets, +the shim applies the restriction between fork and exec, and the +attestation records exactly what ran. When the probe lands below the +venue's expectation the exec still proceeds — recorded and announced, +never silent, never pretended. ``sandbox: off`` runs the command bare +and attests to that honestly. +""" +from __future__ import annotations + +import subprocess +from pathlib import Path + +from lightcone._sandbox_exec import SETUP_FAILURE_EXIT +from lightcone.engine.boundary import ( + BoundaryResult, + ExecScope, + SandboxAttestation, +) +from lightcone.engine.contract import in_container, recipe_env_prefix +from lightcone.engine.sandbox import denial +from lightcone.engine.sandbox import probe as probe_mod +from lightcone.engine.sandbox.policy import EXEC_ALLOWLIST_VERSION, build_policy +from lightcone.engine.sandbox.wrap import run_wrapped, wrap_command + + +class SandboxExecBoundary: + """Enforced recipe execution with honest attestation.""" + + def probe(self, scope: ExecScope) -> SandboxAttestation: + capability = probe_mod.probe() + fs_scope = "project-rw" if scope.writable_project else "declared" + return probe_mod.compose_attestation( + capability, + fs_scope=fs_scope, + exec_allowlist_version=EXEC_ALLOWLIST_VERSION, + disabled=scope.sandbox == "off", + ) + + def execute( + self, + command: str, + scope: ExecScope, + env: dict[str, str], + ) -> BoundaryResult: + capability = probe_mod.probe() + if scope.sandbox == "off": + attestation = probe_mod.compose_attestation( + capability, + fs_scope="open", + exec_allowlist_version=None, + disabled=True, + ) + proc = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + check=False, + cwd=scope.project_root, + env=env, + ) + return BoundaryResult( + returncode=proc.returncode, + stdout=proc.stdout, + stderr=proc.stderr, + attestation=attestation, + ) + + inside = in_container() + policy = build_policy( + scope, + env_prefix=recipe_env_prefix(scope.project_root), + scratch_dirs=self._scratch_dirs(scope), + image_is_exec_set=inside, + ) + wrapped = wrap_command(command, policy, capability) + attestation = probe_mod.compose_attestation( + capability, + fs_scope=policy.fs_scope, + exec_allowlist_version=policy.exec_allowlist_version, + ) + + notes: list[str] = [] + if capability.kind == "none": + # Downgrade below the venue expectation: one console line — + # a user must never finish a run believing they were + # sandboxed when they weren't. + notes.append( + f"\033[33msandbox: no mechanism available " + f"({capability.detail}) — running unsandboxed\033[0m" + ) + + proc = run_wrapped( + wrapped, policy, cwd=scope.project_root, env=env, capture=True + ) + + if proc.returncode == SETUP_FAILURE_EXIT: + # Reserved: sandbox-setup failure — attributed to lc, never + # to the recipe. + notes.append( + "\033[31mlc sandbox setup failed (see above) — this is " + "an lc problem, not the recipe's\033[0m" + ) + elif proc.returncode != 0 and attestation.mechanism != "none": + explanation = denial.explain_failure( + stdout=proc.stdout, + stderr=proc.stderr, + policy=policy, + ) + notes.extend(explanation) + if explanation: + notes.append("") + notes.append(f"\033[2m{denial.trailer(attestation.mechanism)}\033[0m") + + return BoundaryResult( + returncode=proc.returncode, + stdout=proc.stdout, + stderr=proc.stderr, + attestation=attestation, + notes=tuple(notes), + ) + + def describe_host(self) -> str: + return probe_mod.status_line() + + @staticmethod + def _scratch_dirs(scope: ExecScope) -> tuple[Path, ...]: + from lightcone.engine.scratch import resolve_scratch_root + + scratch = resolve_scratch_root(scope.project_root) / ".lightcone" + return (scratch,) if scratch.exists() else () diff --git a/src/lightcone/engine/sandbox/hints.py b/src/lightcone/engine/sandbox/hints.py new file mode 100644 index 00000000..254259fb --- /dev/null +++ b/src/lightcone/engine/sandbox/hints.py @@ -0,0 +1,32 @@ +"""The capped, versioned tool → apt-package hint table. + +Deliberately small: a dozen high-frequency scientific tools, with the +generic ``apt-cache search`` line as the fallback for everything else — +never an open-ended mapping (spec §7). +""" +from __future__ import annotations + +HINT_TABLE_VERSION = 1 + +HINTS: dict[str, str] = { + "latex": "texlive-latex-base", + "pdflatex": "texlive-latex-base", + "xelatex": "texlive-xetex", + "Rscript": "r-base-core", + "R": "r-base-core", + "julia": "julia", + "convert": "imagemagick", + "pdftoppm": "poppler-utils", + "gs": "ghostscript", + "dot": "graphviz", + "ffmpeg": "ffmpeg", + "pandoc": "pandoc", + "gfortran": "gfortran", + "mpirun": "openmpi-bin", +} + + +def apt_hint(tool: str) -> str | None: + """Best-guess apt package for *tool*, or None (caller falls back to + the generic search line).""" + return HINTS.get(tool) diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py new file mode 100644 index 00000000..b65f465f --- /dev/null +++ b/src/lightcone/engine/sandbox/model.py @@ -0,0 +1,47 @@ +"""Data model for the sandbox layer.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + + +@dataclass(frozen=True) +class SandboxPolicy: + """A mechanism-neutral, fully realpath'd enforcement policy. + + Built once per exec by :func:`~lightcone.engine.sandbox.policy.build_policy`; + consumed by the Landlock ruleset builder and the Seatbelt profile + generator alike. + """ + + read: tuple[Path, ...] + write: tuple[Path, ...] + #: Paths granted EXECUTE: the env's bin directory (a dir grant), the + #: enumerated utility binaries, and the realpath'd ELF loaders. + execute: tuple[Path, ...] + tmp_home: Path # fresh per-recipe HOME under the writable tmp scope + env: dict[str, str] # HOME/XDG/MPLCONFIGDIR/PYTHONPYCACHEPREFIX + fs_scope: Literal["declared", "project-rw"] + #: None ⇔ the allowlist did not apply (in-container: the image + #: contents are the exec set). + exec_allowlist_version: int | None + + +@dataclass(frozen=True) +class SandboxCapability: + """What enforcement this host can provide (probed per job).""" + + kind: Literal["landlock", "seatbelt", "none"] + landlock_abi: int | None = None + detail: str = "" + + +@dataclass(frozen=True) +class WrappedCommand: + """A recipe exec, wrapped for enforcement.""" + + argv: tuple[str, ...] + pass_fds: tuple[int, ...] + env: dict[str, str] # additions/overrides for the subprocess env + close_after_spawn: tuple[int, ...] = field(default=()) diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py new file mode 100644 index 00000000..df739cc8 --- /dev/null +++ b/src/lightcone/engine/sandbox/policy.py @@ -0,0 +1,165 @@ +"""The default filesystem policy (spec §7) — one policy, both modes. + +* **write**: the rule's own output dir + the run scratch + ``/tmp`` + + ``/dev/shm`` + ``/dev/null`` + a fresh per-recipe HOME. The + per-output ``writable-project`` escalation adds the project tree and + downgrades the recorded scope to ``project-rw``. Probes (no output + dir) get the tmp scope only — never in-tree. +* **read**: the project tree, declared inputs, and the OS baseline. +* **execute** — two tiers plus the loader: the env's ``bin`` directory, + an enumerated *versioned* utility allowlist, and the realpath'd ELF + loaders — Landlock checks EXECUTE on the interpreter's open of the + loader, so without it every dynamically linked binary (python and + bash included) fails EACCES. Shared libraries need only the read + baseline. +* **HOME/XDG contract**: HOME, ``XDG_{CONFIG,CACHE,DATA}_HOME``, + ``MPLCONFIGDIR``, and ``PYTHONPYCACHEPREFIX`` point at a fresh + per-recipe directory under the writable tmp scope. The real ``$HOME`` + is simply *not granted* — never "fix" a HOME failure by granting it. + (``PYTHONPYCACHEPREFIX`` redirects in-tree bytecode caches to the tmp + scope, eliminating the read-only-tree first-run slowdown without + widening any grant.) +""" +from __future__ import annotations + +import glob +import shutil +import tempfile +from pathlib import Path + +from lightcone.engine.boundary import ExecScope +from lightcone.engine.sandbox.model import SandboxPolicy + +#: Version of the exec allowlist below — recorded in every manifest so +#: an audit can reconstruct exactly what a recipe was allowed to run. +EXEC_ALLOWLIST_VERSION = 1 + +#: v1: shells, the classic text/stream tools, archivers, and a curated +#: coreutils subset. A maintained policy surface — extend by bumping +#: the version, never silently. +EXEC_ALLOWLIST_V1: tuple[str, ...] = ( + "sh", "bash", "env", + "grep", "sed", "awk", "gawk", "mawk", + "tar", "gzip", "gunzip", "zcat", + "cat", "head", "tail", "ls", "cp", "mv", "rm", "mkdir", "rmdir", + "ln", "chmod", "touch", "date", "sort", "uniq", "cut", "tr", "wc", + "tee", "mktemp", "readlink", "realpath", "dirname", "basename", + "echo", "printf", "sleep", "true", "false", +) + +_UTILITY_PATH = "/usr/local/bin:/usr/bin:/bin" + +#: The OS read baseline: interpreters' shared libraries, config, locale +#: and SSL data, /proc self-inspection, entropy. Missing entries are +#: skipped (distro variance), declared inputs are not. +_OS_READ_BASELINE = ( + "/usr", "/lib", "/lib64", "/etc", "/proc", "/sys", "/opt", + "/dev/urandom", "/dev/random", "/dev/zero", "/run", +) + +_ELF_LOADER_GLOBS = ( + "/lib64/ld-linux-*.so.*", + "/lib/ld-linux*.so.*", + "/lib/ld-musl-*.so.*", + "/usr/lib/ld-linux*.so.*", +) + + +def _elf_loaders() -> tuple[Path, ...]: + found: set[Path] = set() + for pattern in _ELF_LOADER_GLOBS: + for hit in glob.glob(pattern): + found.add(Path(hit).resolve()) + return tuple(sorted(found)) + + +def build_policy( + scope: ExecScope, + *, + env_prefix: Path, + scratch_dirs: tuple[Path, ...] = (), + image_is_exec_set: bool = False, +) -> SandboxPolicy: + """Realize the §7 policy for one exec. + + *env_prefix* is the recipe environment's prefix (``/.venv`` + in direct mode, ``/opt/venv`` in an image); its ``bin`` gets a + directory EXECUTE grant. + + *image_is_exec_set* (containerized mode): everything present in the + image WAS declared — the apt layer and extra stage are hashed into + the environment identity — so the whole OS gets the EXECUTE grant + and the utility allowlist is moot. The write/read scoping is + unchanged: in-container Landlock still fences recipes to their own + output. + """ + tmp_home = Path(tempfile.mkdtemp(prefix="lc-home-")) + for sub in (".config", ".cache", ".local/share", ".mplconfig", ".pycache"): + (tmp_home / sub).mkdir(parents=True, exist_ok=True) + + write: list[Path] = [tmp_home] + if scope.output_dir is not None: + scope.output_dir.mkdir(parents=True, exist_ok=True) + write.append(scope.output_dir.resolve()) + write.extend(p.resolve() for p in scratch_dirs if p.exists()) + for p in ("/tmp", "/dev/shm", "/dev/null"): + if Path(p).exists(): + write.append(Path(p).resolve()) + + fs_scope: str = "declared" + if scope.writable_project: + write.append(scope.project_root.resolve()) + fs_scope = "project-rw" + + read: list[Path] = [scope.project_root.resolve()] + read.extend(p.resolve() for p in scope.read_paths if p.exists()) + for p in _OS_READ_BASELINE: + if Path(p).exists(): + read.append(Path(p).resolve()) + + execute: list[Path] = [] + bin_dir = env_prefix / "bin" + if bin_dir.is_dir(): + execute.append(bin_dir.resolve()) + # The venv's `python` is a symlink to the uv-managed interpreter; + # Landlock checks the *resolved* file, so the real install root + # needs read+execute too (realpath every policy path). + python_link = bin_dir / "python" + if python_link.exists(): + real = python_link.resolve() + install_root = real.parent.parent + execute.append(install_root) + read.append(install_root) + if image_is_exec_set: + for p in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt"): + if Path(p).exists(): + execute.append(Path(p).resolve()) + else: + for name in EXEC_ALLOWLIST_V1: + if hit := shutil.which(name, path=_UTILITY_PATH): + execute.append(Path(hit).resolve()) + execute.extend(_elf_loaders()) + + env = { + "HOME": str(tmp_home), + "XDG_CONFIG_HOME": str(tmp_home / ".config"), + "XDG_CACHE_HOME": str(tmp_home / ".cache"), + "XDG_DATA_HOME": str(tmp_home / ".local/share"), + "MPLCONFIGDIR": str(tmp_home / ".mplconfig"), + "PYTHONPYCACHEPREFIX": str(tmp_home / ".pycache"), + } + + return SandboxPolicy( + read=tuple(dict.fromkeys(read)), + write=tuple(dict.fromkeys(write)), + execute=tuple(dict.fromkeys(execute)), + tmp_home=tmp_home, + env=env, + fs_scope=fs_scope, # type: ignore[arg-type] + # In-container the image contents are the exec set — recording + # an allowlist version there would claim a policy that didn't + # apply. + exec_allowlist_version=( + None if image_is_exec_set else EXEC_ALLOWLIST_VERSION + ), + ) diff --git a/src/lightcone/engine/sandbox/probe.py b/src/lightcone/engine/sandbox/probe.py new file mode 100644 index 00000000..0fed1444 --- /dev/null +++ b/src/lightcone/engine/sandbox/probe.py @@ -0,0 +1,120 @@ +"""Capability probe + hermeticity composition. + +The probe runs worker-side, per job — the driver's kernel is not the +worker's, and in containerized mode the relevant question (are the +``landlock_*`` syscalls admitted by the seccomp profile?) can only be +answered inside the container. :func:`compose_attestation` is the single +home of the §7 enum mapping; the manifest records what the *applied +flags* were, never a documentation row. +""" +from __future__ import annotations + +import os +import platform +import subprocess +from functools import cache + +from lightcone.engine.boundary import SandboxAttestation +from lightcone.engine.contract import CONTAINER_NETWORK_ENV, in_container +from lightcone.engine.sandbox.model import SandboxCapability + + +@cache +def probe() -> SandboxCapability: + system = platform.system() + if system == "Linux": + from lightcone.engine.sandbox import _landlock + + abi = _landlock.abi() + if abi > 0: + return SandboxCapability(kind="landlock", landlock_abi=abi) + return SandboxCapability( + kind="none", + detail="landlock unavailable (kernel < 5.13, or seccomp blocks it)", + ) + if system == "Darwin": + return _seatbelt_capability() + return SandboxCapability(kind="none", detail=f"no sandbox mechanism on {system}") + + +def _seatbelt_capability() -> SandboxCapability: + if not os.path.isfile("/usr/bin/sandbox-exec"): + return SandboxCapability(kind="none", detail="sandbox-exec not present") + try: + canary = subprocess.run( + ["/usr/bin/sandbox-exec", "-p", "(version 1)(allow default)", + "/usr/bin/true"], + capture_output=True, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as e: + return SandboxCapability(kind="none", detail=f"sandbox-exec canary failed: {e}") + if canary.returncode != 0: + return SandboxCapability( + kind="none", + detail=f"sandbox-exec canary exited {canary.returncode} " + f"(macOS {platform.mac_ver()[0]})", + ) + return SandboxCapability(kind="seatbelt") + + +def compose_attestation( + capability: SandboxCapability, + *, + fs_scope: str, + exec_allowlist_version: int | None, + disabled: bool = False, +) -> SandboxAttestation: + """The §7 hermeticity record for one exec. + + Network enum, normative mapping: Landlock cannot express a useful + deny (ABI ≤ 3 has no network control; ABI 4 cannot carve out + loopback) ⇒ ``unenforced``; Seatbelt denies non-loopback ⇒ + ``denied``; a container run under ``--net=none`` ⇒ ``denied``; + no restriction applied ⇒ ``allowed``. + """ + inside = in_container() + container_network = os.environ.get(CONTAINER_NETWORK_ENV) + + if disabled or capability.kind == "none": + if inside: + # The mount set still bounds the world even without an + # in-container Landlock tier. + return SandboxAttestation( + mechanism="podman", + fs="project-rw", + network="denied" if container_network == "none" else "allowed", + ) + return SandboxAttestation(mechanism="none", fs="open", network="allowed") + + if capability.kind == "landlock": + mechanism = "podman+landlock" if inside else "landlock" + if inside: + network = "denied" if container_network == "none" else "allowed" + else: + network = "unenforced" + return SandboxAttestation( + mechanism=mechanism, + fs=fs_scope, + network=network, + landlock_abi=capability.landlock_abi, + exec_allowlist_version=exec_allowlist_version, + ) + + # seatbelt + return SandboxAttestation( + mechanism="seatbelt", + fs=fs_scope, + network="denied", + exec_allowlist_version=exec_allowlist_version, + ) + + +def status_line() -> str: + """The ``lc status`` sandbox header line for this host.""" + cap = probe() + att = compose_attestation( + cap, fs_scope="declared", exec_allowlist_version=None + ) + detail = f" — {cap.detail}" if cap.kind == "none" and cap.detail else "" + return f"{att.mechanism} (fs: {att.fs}, network: {att.network}){detail}" diff --git a/src/lightcone/engine/sandbox/seatbelt.py b/src/lightcone/engine/sandbox/seatbelt.py new file mode 100644 index 00000000..342f09b9 --- /dev/null +++ b/src/lightcone/engine/sandbox/seatbelt.py @@ -0,0 +1,71 @@ +"""Seatbelt (macOS) profile generation. + +The generated SBPL realizes the same :class:`SandboxPolicy` Landlock +enforces on Linux: deny by default; read/write/exec sets realpath'd; +POSIX shm/semaphores allowed (multiprocessing); network denied except +loopback — the spec's meaning of ``denied`` keeps in-recipe +LocalCluster/torch workers working. + +``sandbox-exec`` cannot nest — a recipe invoking it fails by design +(the denial UX names it plainly). The profile shape is pinned by a +golden test on Linux; enforcement is smoke-tested on macOS CI. +""" +from __future__ import annotations + +from lightcone.engine.sandbox.model import SandboxPolicy + +#: System paths every process needs readable (dyld, frameworks, ICU). +_MACOS_READ_BASELINE = ( + "/System", + "/usr/lib", + "/usr/share", + "/private/etc", + "/Library/Preferences/Logging", + "/dev/urandom", + "/dev/random", + "/dev/null", +) + + +def _quote(path: object) -> str: + return '"' + str(path).replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def generate_profile(policy: SandboxPolicy) -> str: + read_paths = [*(_MACOS_READ_BASELINE), *policy.read] + lines: list[str] = [ + ";; generated by lightcone-cli — the lc sandbox (Seatbelt realization)", + "(version 1)", + "(deny default)", + "(allow process-fork)", + "(allow sysctl-read)", + "(allow mach-lookup)", + "(allow ipc-posix-shm*)", + "(allow ipc-posix-sem*)", + "(allow signal (target children))", + "", + ";; read set: project + declared inputs + OS baseline", + "(allow file-read*", + *(f" (subpath {_quote(p)})" for p in read_paths), + ")", + "", + ";; write set: own output + scratch + tmp + per-recipe HOME", + "(allow file-write* file-read*", + *(f" (subpath {_quote(p)})" for p in policy.write), + ")", + "", + ";; exec set: env bin dir + versioned utility allowlist", + "(allow process-exec*", + *(f" (subpath {_quote(p)})" for p in policy.execute), + " (literal \"/usr/lib/dyld\")", + " (literal \"/bin/sh\")", + " (literal \"/bin/bash\")", + ")", + "", + ";; network: loopback only — 'denied' means non-loopback blocked", + "(allow network-outbound (remote ip \"localhost:*\"))", + "(allow network-bind network-inbound (local ip \"localhost:*\"))", + "(deny network-outbound (remote ip \"*:*\"))", + "", + ] + return "\n".join(lines) diff --git a/src/lightcone/engine/sandbox/wrap.py b/src/lightcone/engine/sandbox/wrap.py new file mode 100644 index 00000000..9dcf2c60 --- /dev/null +++ b/src/lightcone/engine/sandbox/wrap.py @@ -0,0 +1,182 @@ +"""Assemble the wrapped recipe exec. + +The boundary placement is the **exec-shim** (spec §7): the sandbox +wraps *the recipe*, not the engine. The parent builds the Landlock +ruleset FD before fork and passes it down (``pass_fds`` + env); the +shim (:mod:`lightcone._sandbox_exec`) performs exactly +``prctl(PR_SET_NO_NEW_PRIVS)`` + ``landlock_restrict_self(fd)`` and +execs bash. No ``preexec_fn`` anywhere. + +The shim is launched with the *current interpreter* — ``run_rule`` +already executes inside the recipe environment (the delegated engine in +direct mode, the image's ``/opt/venv`` in containerized mode), so no +``uv run`` hop is needed. (The FD-survival spike verified inheritance +holds even through uv's spawn chain; the direct exec makes it moot.) +""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +from lightcone._sandbox_exec import ( + SANDBOX_FD_ENV, + SANDBOX_MODE_ENV, + SANDBOX_PROFILE_ENV, +) +from lightcone.engine.sandbox.model import ( + SandboxCapability, + SandboxPolicy, + WrappedCommand, +) + + +def wrap_command( + shell_command: str, + policy: SandboxPolicy, + capability: SandboxCapability, +) -> WrappedCommand: + """Wrap a rule's shell command (the ``run_rule`` path).""" + return wrap_argv( + ("bash", "-c", shell_command), + policy, + capability, + interpreter=(sys.executable,), + ) + + +def wrap_argv( + recipe_argv: tuple[str, ...], + policy: SandboxPolicy, + capability: SandboxCapability, + *, + interpreter: tuple[str, ...], +) -> WrappedCommand: + """Wrap an arbitrary argv through the shim. + + *interpreter* is how the shim's python is reached — the current + interpreter for rules, or a ``uv run … -- python`` prefix for + probes (uv stays trusted plumbing *outside* the boundary; the FD + survives its spawn chain — spike-verified). + """ + argv = ( + *interpreter, + "-m", + "lightcone._sandbox_exec", + "--", + *recipe_argv, + ) + + if capability.kind == "landlock": + fd = _build_ruleset(policy) + return WrappedCommand( + argv=argv, + pass_fds=(fd,), + env={ + SANDBOX_MODE_ENV: "landlock", + SANDBOX_FD_ENV: str(fd), + **policy.env, + }, + close_after_spawn=(fd,), + ) + + if capability.kind == "seatbelt": + from lightcone.engine.sandbox.seatbelt import generate_profile + + with tempfile.NamedTemporaryFile( + "w", prefix="lc-sandbox-", suffix=".sb", delete=False + ) as f: + f.write(generate_profile(policy)) + profile_path = f.name + return WrappedCommand( + argv=argv, + pass_fds=(), + env={ + SANDBOX_MODE_ENV: "seatbelt", + SANDBOX_PROFILE_ENV: profile_path, + **policy.env, + }, + ) + + return WrappedCommand( + argv=argv, + pass_fds=(), + env={SANDBOX_MODE_ENV: "none", **policy.env}, + ) + + +def run_wrapped( + wrapped: WrappedCommand, + policy: SandboxPolicy, + *, + cwd: Path, + env: dict[str, str], + capture: bool, +) -> subprocess.CompletedProcess[str]: + """Spawn a wrapped exec and uphold its lifecycle invariants — close + the ruleset FD after spawn, reclaim the per-recipe HOME — in the one + place the rule path and the probe path share. With ``capture=False`` + stdio is inherited (interactive probes/shells) and the returned + process carries no output. + """ + try: + return subprocess.run( + list(wrapped.argv), + capture_output=capture, + text=True, + check=False, + cwd=cwd, + env={**env, **wrapped.env}, + pass_fds=wrapped.pass_fds, + ) + finally: + for fd in wrapped.close_after_spawn: + try: + os.close(fd) + except OSError: + pass + shutil.rmtree(policy.tmp_home, ignore_errors=True) + + +def _build_ruleset(policy: SandboxPolicy) -> int: + """Build the Landlock ruleset FD from the policy (parent side).""" + import os + + from lightcone.engine.sandbox import _landlock + + abi = _landlock.abi() + handled = _landlock.handled_access_for(abi) + fd = _landlock.create_ruleset(handled) + try: + read_bits = _landlock.READ_BITS + write_bits = _landlock.write_bits(abi) | read_bits + exec_bits = _landlock.ACCESS_FS_EXECUTE | read_bits + for path in policy.read: + _add_if_exists(fd, path, read_bits) + for path in policy.write: + _add_if_exists(fd, path, write_bits) + for path in policy.execute: + _add_if_exists(fd, path, exec_bits) + except BaseException: + os.close(fd) + raise + # The FD must survive fork+exec into the shim. + os.set_inheritable(fd, True) + return fd + + +def _add_if_exists(fd: int, path: object, access: int) -> None: + from pathlib import Path + + from lightcone.engine.sandbox import _landlock + + p = Path(str(path)) + try: + _landlock.add_path_rule(fd, p, access) + except FileNotFoundError: + # OS-baseline entries vary by distro; a vanished path grants + # nothing, which is safe (allowlist semantics). + pass diff --git a/src/lightcone/engine/scratch.py b/src/lightcone/engine/scratch.py index 0b3d0cf0..7746ca43 100644 --- a/src/lightcone/engine/scratch.py +++ b/src/lightcone/engine/scratch.py @@ -2,24 +2,15 @@ A single concept: where lightcone keeps its operational state — snakemake metadata, dask worker spill, the run-exclusion lock. Resolved at the -start of every ``lc run``. Resolution precedence (first hit wins): +start of every ``lc materialize``. Resolution precedence (first hit wins): 1. ``LIGHTCONE_SCRATCH`` env var (escape hatch / CI override). 2. ``scratch_root`` in ``/.lightcone/lightcone.yaml`` (per-project pin). -3. ``scratch_root`` from the detected site in - :mod:`lightcone.engine.site_registry`. Stored as a shell expression - (e.g. ``$SCRATCH``) and expanded with :func:`os.path.expandvars`. -4. :func:`tempfile.gettempdir` fallback (single-node only). +3. :func:`tempfile.gettempdir` fallback. The resolved path is then used as the parent of ``.lightcone/`` — multiple projects can share one scratch root without colliding because snakemake state is keyed by a hash of the project's absolute path. - -Why this matters on NERSC: ``$HOME`` and ``/global/cfs`` are mounted on -compute nodes via DVS, which `does not support file locking -`_. Snakemake's workflow -lock, our run-exclusion lock, and any future coordination primitive -silently fail there. ``$SCRATCH`` is Lustre, which works correctly. """ from __future__ import annotations @@ -35,8 +26,6 @@ import yaml -from lightcone.engine.site_registry import detect_current_site - LIGHTCONE_SCRATCH_ENV = "LIGHTCONE_SCRATCH" @@ -72,13 +61,6 @@ def resolve_scratch_root(project_path: Path) -> Path: if val := data.get("scratch_root"): return Path(os.path.expandvars(str(val))).expanduser() - if val := detect_current_site().get("scratch_root"): - expanded = os.path.expandvars(str(val)) - # ``$VAR`` left intact means the env wasn't set — don't write - # to a literal path called ``$SCRATCH``. Fall through. - if not expanded.startswith("$") and "$" not in expanded: - return Path(expanded).expanduser() - return Path(tempfile.gettempdir()) diff --git a/src/lightcone/engine/site_registry.py b/src/lightcone/engine/site_registry.py deleted file mode 100644 index 9fd282dd..00000000 --- a/src/lightcone/engine/site_registry.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Known site defaults. - -When ``lc init`` runs on a known site, the matching entry below provides -the scratch root surfaced to the user and any deny rules used to keep -edits off shared filesystems. - -To add a new site, append an entry to :data:`SITE_DEFAULTS`. - -The high-level entry point for the rest of the codebase is -:func:`detect_current_site`, which returns a :class:`HostSite` bundling -the matched site key with its declared defaults — keeping the -``socket.gethostname() + detect_site + get_site_defaults`` chain in one -place. -""" -from __future__ import annotations - -import os -import socket -from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any - -#: Per-site defaults. ``suggested_options`` follows the same shape as the -#: target file's ``options`` section: an orthogonal map of axis → -#: ``{default, choices}`` (where ``choices`` is ``{value: guidance}``). -#: ``cache_key_overrides`` captures non-conventional sacctmgr naming (e.g. -#: Perlmutter's ``regular_1`` for the CPU ``regular`` queue). -SITE_DEFAULTS: dict[str, dict[str, Any]] = { - "perlmutter": { - "hostname_patterns": ["perlmutter", "saul"], - "display_name": "NERSC Perlmutter", - "backend": "slurm", - "connection": { - "hostname": "perlmutter.nersc.gov", - }, - "container_runtime": "podman-hpc", - # Where lightcone keeps its operational state (snakemake metadata, - # dask spill, cross-node stdout locks). NERSC's $HOME and CFS are - # mounted on compute via DVS, which silently swallows ``flock`` and - # is slow for small-file I/O — Lustre ($SCRATCH) is the only sane - # choice. Stored as a shell expression so it expands to each user's - # private scratch path at run time. - "scratch_root": "$SCRATCH", - "suggested_options": { - "qos": { - "default": "debug", - "choices": { - "debug": "quick iteration, testing", - "regular": "production runs, large jobs", - "preempt": "cheap batch, restartable after 2h", - "shared": "fractional node (1–2 GPUs)", - }, - }, - "constraint": { - "default": "gpu", - "choices": { - "gpu": "A100 40 GB — 1,536 nodes, 4 GPUs/node", - "cpu": "CPU only — 3,072 nodes, 128 cores/node", - "gpu&hbm80g": "A100 80 GB — 256 nodes", - }, - }, - "time_limit": { - "default": "30m", - "guidance": "debug caps at 30 min; regular allows up to 48 h", - }, - }, - # Perlmutter's sacctmgr names prefix GPU QoS with `gpu_` and - # suffix the CPU regular queue as `regular_1`. The first is - # handled by the default `{constraint}_{qos}` convention; the - # second needs an explicit override. - "cache_key_overrides": { - "regular/cpu": "regular_1", - }, - "scratch_paths": [ - "//pscratch/**", - "//global/cscratch1/**", - "//global/cfs/cdirs/**", - ], - }, - # A JupyterHub deployment with Dask Gateway (e.g. lightcone-hub on - # GKE). Unlike HPC sites, hostnames here are meaningless pod names — - # detection is by the env vars the deployment injects into every - # user pod. ``container_runtime: kubernetes`` routes recipe - # execution through worker pods running the project image. - # - # ``scratch_root`` must be declared even though "local" gets by - # without one: with no site scratch, resolution falls back to the - # tempdir — fine on a single machine, but a pod's ``/tmp`` is - # pod-local. The ``.snakemake`` state the driver symlinks into - # scratch has to live on the NFS home every worker pod mounts, or - # each worker would write its job metadata into its own ``/tmp`` - # (invisible to the driver) and every subsequent run would consider - # all outputs stale. ``$HOME`` *is* the shared filesystem here. - "jupyterhub": { - "hostname_patterns": [], - "env_markers": ["DASK_GATEWAY__ADDRESS"], - "display_name": "JupyterHub (Dask Gateway)", - "backend": "kubernetes", - "connection": {}, - "container_runtime": "kubernetes", - "scratch_root": "$HOME", - }, - "local": { - "hostname_patterns": [], - "display_name": "Local", - "backend": "local", - "connection": {}, - }, -} - - -def detect_site(hostname_or_name: str) -> str | None: - """Detect a known site from a hostname or site name.""" - normalized = hostname_or_name.lower() - for site_key, site in SITE_DEFAULTS.items(): - if site.get("backend") == "local": - continue - if site_key in normalized: - return site_key - for pattern in site.get("hostname_patterns", []): - if pattern in normalized: - return site_key - return None - - -def detect_site_from_env() -> str | None: - """Detect a site whose declared ``env_markers`` are all present. - - Deployment-style sites (JupyterHub pods) have arbitrary hostnames; - what identifies them is the environment the deployment injects. - """ - for site_key, site in SITE_DEFAULTS.items(): - markers = site.get("env_markers") or [] - if markers and all(os.environ.get(m) for m in markers): - return site_key - return None - - -def get_site_defaults(site_key: str) -> dict[str, Any] | None: - """Return defaults for a known site, or ``None``.""" - return SITE_DEFAULTS.get(site_key) - - -def list_known_sites() -> list[tuple[str, str]]: - """Return ``(site_key, display_name)`` for all known sites.""" - return [ - (key, site.get("display_name", key)) - for key, site in SITE_DEFAULTS.items() - ] - - -def get_site_scratch_deny_rules(site_key: str) -> list[str]: - """Return Edit deny rules for a site's scratch/shared filesystems.""" - site = SITE_DEFAULTS.get(site_key) - if not site: - return [] - scratch_paths = site.get("scratch_paths", []) - return [f"Edit({path})" for path in scratch_paths] - - -@dataclass(frozen=True) -class HostSite: - """The site (if any) the local host belongs to. - - Returned by :func:`detect_current_site`. Use ``if site:`` to test - whether a known site was matched; use :meth:`get` (or - :attr:`defaults`) to read declared fields. - - Adding a new "site asks for X" feature should not require a fourth - copy of the ``detect_site(socket.gethostname()) → get_site_defaults`` - boilerplate — extend this class (or its consumers) instead. - """ - - key: str | None - defaults: Mapping[str, Any] = field(default_factory=dict) - - def __bool__(self) -> bool: - return self.key is not None - - @property - def display_name(self) -> str: - return self.defaults.get("display_name") or self.key or "unknown" - - def get(self, name: str, default: Any = None) -> Any: - """Look up a field declared in the site's defaults.""" - return self.defaults.get(name, default) - - -_UNKNOWN_HOST_SITE = HostSite(key=None, defaults={}) - - -def detect_current_site() -> HostSite: - """Return the :class:`HostSite` for the local host. - - Single source of truth for "which site are we on?" — everything else - in the codebase should call this rather than re-deriving it from - :func:`socket.gethostname` and :func:`detect_site`. Environment - markers win over hostname patterns (a pod's hostname is noise; the - injected env is the signal). Returns a falsy :class:`HostSite` - (``key is None``) when nothing matches. - """ - key = detect_site_from_env() or detect_site(socket.gethostname()) - if key is None: - return _UNKNOWN_HOST_SITE - return HostSite(key=key, defaults=get_site_defaults(key) or {}) diff --git a/src/lightcone/engine/snakefile.py b/src/lightcone/engine/snakefile.py index 7bc77010..eeac2705 100644 --- a/src/lightcone/engine/snakefile.py +++ b/src/lightcone/engine/snakefile.py @@ -3,17 +3,19 @@ The Snakefile is a thin shell over the astra spec: one rule per output with a recipe, parameterized by ``{universe}``. Each rule's body is a ``run:`` block that calls :func:`lightcone.engine.runner.run_rule` with -the per-(rule, universe) cfg blob — which already contains the rendered -and wrapped ``shell_command``. All template substitution and container -wrapping happen here, at generation time, where every value is concrete -for a given universe; the runner stays a thin executor. - -ASTRA v0.0.7 moved ``inputs`` and ``decisions`` declarations from -``Recipe`` up to ``Output``. The recipe body is a *template* using a -small placeholder grammar — see :func:`render_recipe`. We don't use -Snakemake's ``container:`` directive or ``--sdm apptainer``: the -generator wraps with the configured runtime end-to-end (see -:mod:`lightcone.engine.container`). +the per-(rule, universe) cfg blob. All template substitution happens +here, at generation time, where every value is concrete for a given +universe; the runner stays a thin executor. + +Recipes are **never wrapped** at generation time: enforcement (the +sandbox) is applied at exec time by the boundary +(:mod:`lightcone.engine.boundary`), and in containerized mode the +entire stack already runs inside the project image — wrapping recipes +individually would containerize twice. + +ASTRA carries only analysis structure (inputs/outputs/recipes/ +decisions/universes) — the environment lives in ``pyproject.toml`` + +``uv.lock``. Legacy ``container:`` keys in astra.yaml are ignored. The ``os.replace`` rename inside ``write_manifest`` (called by ``run_rule``) is the atomic commit point — either the rule produced @@ -29,19 +31,24 @@ from astra.helpers import load_yaml, resolve_analysis_tree -from lightcone.engine.container import ( - make_image_tag_resolver, - runtime_registry, - wrap_recipe, +from lightcone.engine import lc_version +from lightcone.engine.environment import ( + EnvironmentSpec, + LockScan, + Mode, + ProjectEnvironmentError, + load_environment, + scan_lock, ) -from lightcone.engine.manifest import code_version +from lightcone.engine.job import RuleJob +from lightcone.engine.manifest import MANIFEST_FILENAME, code_version from lightcone.engine.tree import ( TreeOutput, collect_tree_outputs, find_upstream_output, - resolve_container_spec, + load_universe_decisions, resolve_external_input, - resolve_universe_decisions, + scoped_decisions_for_output, ) LIGHTCONE_DIR = ".lightcone" @@ -135,6 +142,23 @@ def _git_sha(project_path: Path) -> str | None: return None +def _git_dirty(project_path: Path) -> bool | None: + """True when the working tree differs from HEAD — a manifest whose + ``git_sha`` cannot fully reproduce the run. ``None`` outside git.""" + try: + out = subprocess.run( + ["git", "-C", str(project_path), "status", "--porcelain"], + capture_output=True, + text=True, + check=False, + ) + if out.returncode == 0: + return bool(out.stdout.strip()) + except FileNotFoundError: + pass + return None + + def _git_remote(project_path: Path) -> str | None: """URL of the ``origin`` git remote, if the project is a git clone. @@ -167,15 +191,6 @@ def _git_remote(project_path: Path) -> str | None: return None -def _lc_version() -> str: - try: - from importlib.metadata import version - - return version("lightcone-cli") - except Exception: - return "unknown" - - def _output_dir_pattern(tree_out: TreeOutput) -> str: """Wildcard path to this output's directory. @@ -188,18 +203,10 @@ def _output_dir_pattern(tree_out: TreeOutput) -> str: return f"results/{{universe}}/{tree_out.output_id}" -def _rule_key(tree_out: TreeOutput) -> str: - """Unique key into the cfg JSON. Avoids collisions when two - sub-analyses share an output_id.""" - if tree_out.analysis_id is None: - return tree_out.output_id - return f"{tree_out.analysis_id}.{tree_out.output_id}" - - def _rule_name(tree_out: TreeOutput) -> str: - """Snakemake rule name. Mirrors the cfg key but with + """Snakemake rule name. Mirrors ``qualified_id`` but with Snakemake-friendly identifier characters (``.`` → ``__``).""" - return _rule_key(tree_out).replace(".", "__") + return tree_out.qualified_id.replace(".", "__") def _safe_input_key(raw_id: str) -> str: @@ -212,43 +219,6 @@ def _safe_input_key(raw_id: str) -> str: return raw_id.replace(".", "__") -def _scoped_decisions_for_output( - tree_out: TreeOutput, - universe_decisions: dict[str, Any], -) -> dict[str, Any]: - """Pick the active option ID for each decision the Output declares. - - v0.0.7: ``Output.decisions`` lists the IDs of decisions that - parameterize this output. The runner only needs (and the recipe - template can only reference) those — anything else is out of scope. - """ - declared = tree_out.output_def.get("decisions") or [] - if not declared: - return {} - scoped: dict[str, Any] = {} - prefix = f"{tree_out.analysis_id}." if tree_out.analysis_id else "" - for dec_id in declared: - if prefix and (qualified := f"{prefix}{dec_id}") in universe_decisions: - scoped[dec_id] = universe_decisions[qualified] - elif dec_id in universe_decisions: - scoped[dec_id] = universe_decisions[dec_id] - return scoped - - -def _universe_decisions( - universe_id: str, - project_path: Path, - spec: dict[str, Any], -) -> dict[str, Any]: - universe_yaml = project_path / "universes" / f"{universe_id}.yaml" - if not universe_yaml.exists(): - return {} - try: - return resolve_universe_decisions(project_path, spec, universe_id) - except (FileNotFoundError, KeyError): - return {} - - def _render_snakefile( rules: list[dict[str, Any]], universes: list[str], @@ -271,7 +241,7 @@ def _render_snakefile( rule_all_inputs = [] for r in rules: rule_all_inputs.append( - f' expand("{r["output_dir"]}/.lightcone-manifest.json", ' + f' expand("{r["output_dir"]}/{MANIFEST_FILENAME}", ' f"universe=UNIVERSES)," ) rule_all_block = "\n".join(rule_all_inputs) or " []" @@ -301,7 +271,7 @@ def _render_snakefile( lines.append(f' {safe}="{pattern}",') lines.append(" output:") lines.append(f' data=directory("{r["output_dir"]}"),') - lines.append(f' manifest="{r["output_dir"]}/.lightcone-manifest.json",') + lines.append(f' manifest="{r["output_dir"]}/{MANIFEST_FILENAME}",') lines.append(" params:") lines.append(f' cfg=lambda wc: CFG["{r["key"]}"][wc.universe],') lines.append(" run:") @@ -328,46 +298,85 @@ def generate( project_path: Path, *, universes: list[str], - runtime: str = "none", + env: EnvironmentSpec | None = None, + scan: LockScan | None = None, ) -> tuple[Path, Path]: """Write ``.lightcone/Snakefile`` and ``.lightcone/snakefile-config.json``. Args: project_path: Project root containing ``astra.yaml``. universes: Universe ids to expand rules over. - runtime: Container runtime to wrap recipes with. One of - ``docker | podman | podman-hpc | kubernetes | none``. - ``none`` runs recipes on the host without isolation; - ``kubernetes`` leaves recipes unwrapped (the worker pod runs - the project image) and resolves Containerfile specs to - registry refs. Resolution is done here once, not per-rule, - so all rules use a consistent runtime. See - :func:`lightcone.engine.container.load_runtime`. + env: The loaded project environment (loaded here when omitted). + Its ``env_version`` flows into every rule's ``code_version`` + and cfg — the identity every worker's mid-run gates re-check. + + Raises :class:`~lightcone.engine.environment.ProjectEnvironmentError` + when the lock scan refuses (unauditable path/editable dependencies). Returns ``(snakefile_path, config_path)``. """ project_path = Path(project_path).resolve() + if env is None: + env = load_environment(project_path) + + if scan is None: + scan = scan_lock(project_path) + if scan.refusals: + raise ProjectEnvironmentError( + "uv.lock contains unauditable dependencies:\n " + + "\n ".join(scan.refusals) + + "\nPin them to a registry (or vendor them as declared inputs)." + ) + spec = resolve_analysis_tree(load_yaml(project_path / "astra.yaml"), project_path) - project_name = (spec.get("name") or project_path.name).lower().replace(" ", "-") tree_outputs = collect_tree_outputs(spec) + # Validate the per-output sandbox escalations against declared ids. + declared_ids = {to.qualified_id for to in tree_outputs} + if unknown := env.writable_project_outputs - declared_ids: + raise ProjectEnvironmentError( + "[tool.lightcone.sandbox] writable-project names undeclared " + f"output(s): {', '.join(sorted(unknown))}." + ) + rules: list[dict[str, Any]] = [] cfg: dict[str, dict[str, dict[str, Any]]] = {} git_sha = _git_sha(project_path) + git_dirty = _git_dirty(project_path) git_remote = _git_remote(project_path) - lc_version = _lc_version() - resolve_image = make_image_tag_resolver( - project_path, project_name, registry=runtime_registry(runtime) - ) + version = lc_version() + # Universe decisions depend only on (universe, spec) — resolve each + # once, not per rule (the resolution re-reads universe YAMLs). + decisions_by_universe = { + u: load_universe_decisions(project_path, spec, u) for u in universes + } + + # Containerized mode: pin every job to the image the driver resolved + # — the digest travels in the job command and run_rule asserts it. + image_tag: str | None = None + image_digest: str | None = None + dpkg_snapshot_sha256: str | None = None + if env.mode is Mode.CONTAINERIZED: + from lightcone.engine.image import read_record + + record = read_record(project_path) + if record is None or record.env_version != env.env_version: + raise ProjectEnvironmentError( + "the environment image is not built for the current " + "environment — run `lc build`." + ) + image_tag = record.tag + image_digest = record.image_id + dpkg_snapshot_sha256 = record.dpkg_snapshot_sha256 for to in tree_outputs: recipe = to.output_def.get("recipe") if recipe is None: continue # alias output (re-export via ``from:``) - rule_key = _rule_key(to) + rule_key = to.qualified_id rule_name = _rule_name(to) out_dir_pattern = _output_dir_pattern(to) @@ -392,8 +401,7 @@ def generate( pattern = ext rule_inputs.append((inp_id, _safe_input_key(inp_id), pattern)) - container_image = resolve_container_spec(to, spec) - image_tag = resolve_image(container_image) + writable_project = rule_key in env.writable_project_outputs rules.append( { @@ -406,8 +414,9 @@ def generate( cfg.setdefault(rule_key, {}) for u in universes: - universe_decisions = _universe_decisions(u, project_path, spec) - scoped_decisions = _scoped_decisions_for_output(to, universe_decisions) + scoped_decisions = scoped_decisions_for_output( + to, decisions_by_universe[u] + ) # Build the resolved input map in declaration order so # ``{inputs}`` joins paths in the same order the user wrote @@ -424,38 +433,43 @@ def generate( decisions=scoped_decisions, output=output_dir, ) - wrapped = wrap_recipe(rendered, image=image_tag, runtime=runtime) - # ``image_tag`` (not the raw spec string) so a Containerfile - # edit propagates through ``code_version`` to ``lc status``. cv = code_version( recipe=recipe_command, - container_image=image_tag, decisions=scoped_decisions, + env_version=env.env_version, + writable_project=writable_project, ) # Prefix the executed command with a no-op ``:`` builtin - # carrying the code_version. This makes the wrapped command - # differ when the recipe / container / decisions drift, so + # carrying the code_version. This makes the command differ + # when the recipe / environment / decisions drift, so # (a) Snakemake's ``shellcmd`` trigger sees the change and # (b) any shell trace carries a breadcrumb. The trigger # that actually fires today is ``params`` (cfg is # per-universe and contains ``shell_command``) — see - # ``lc run --rerun-triggers``. - shell_command = f": lc_code_version={cv};\n{wrapped}" - cfg[rule_key][u] = { - "output_id": to.output_id, - "output_type": to.output_def.get("type"), - "universe_id": u, - # Raw template, preserved so the manifest's ``recipe`` - # field records what the user authored. - "recipe": recipe_command, - "shell_command": shell_command, - "container_image": container_image, - "decisions": scoped_decisions, - "code_version": cv, - "git_sha": git_sha, - "git_remote": git_remote, - "lc_version": lc_version, - } + # ``lc materialize --rerun-triggers``. + shell_command = f": lc_code_version={cv};\n{rendered}" + cfg[rule_key][u] = RuleJob( + output_id=to.output_id, + output_type=to.output_def.get("type"), + universe_id=u, + recipe=recipe_command, + shell_command=shell_command, + decisions=scoped_decisions, + code_version=cv, + env_version=env.env_version, + writable_project=writable_project, + sdist_built=list(scan.sdist_built), + git_sha=git_sha, + git_dirty=git_dirty, + git_remote=git_remote, + lc_version=version, + worker_runtime=( + "container" if env.mode is Mode.CONTAINERIZED else "host" + ), + image_tag=image_tag, + image_digest=image_digest, + dpkg_snapshot_sha256=dpkg_snapshot_sha256, + ).to_cfg() lightcone_dir = project_path / LIGHTCONE_DIR lightcone_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/lightcone/engine/status.py b/src/lightcone/engine/status.py index 8175420a..bb14723c 100644 --- a/src/lightcone/engine/status.py +++ b/src/lightcone/engine/status.py @@ -1,11 +1,14 @@ """Manifest-driven status walker. -For each output declared in a project's ``astra.yaml``, determines whether -it is materialized, stale, missing, or an alias — by reading the per-output -manifest written at ``/.lightcone-manifest.json``. - -This module never imports Snakemake. ``lc status`` works on a fresh clone -with no ``.snakemake/`` directory and on frozen archives. +For each output declared in a project's ``astra.yaml``, determines +whether it is materialized, stale, missing, pre-migration, or an alias — +by reading the per-output manifest at +``/.lightcone-manifest.json``. + +Offline and local-only by invariant: this module reads the project tree +(spec, manifests, ``pyproject.toml``) and never the network. It never +imports Snakemake — ``lc status`` works on a fresh clone with no +``.snakemake/`` directory and on frozen archives. """ from __future__ import annotations @@ -16,21 +19,16 @@ from astra.helpers import load_yaml, resolve_analysis_tree -from lightcone.engine.container import ( - load_runtime, - make_image_tag_resolver, - runtime_registry, -) -from lightcone.engine.manifest import code_version, read_manifest +from lightcone.engine.environment import EnvironmentSpec, load_environment +from lightcone.engine.manifest import code_version, is_pre_migration, read_manifest from lightcone.engine.tree import ( - TreeOutput, collect_tree_outputs, - resolve_container_spec, + load_universe_decisions, resolve_output_path, - resolve_universe_decisions, + scoped_decisions_for_output, ) -StatusLiteral = Literal["ok", "stale", "missing", "alias"] +StatusLiteral = Literal["ok", "stale", "missing", "alias", "pre_migration"] @dataclass @@ -44,66 +42,30 @@ class OutputStatus: recipe_command: str | None -def _decisions_for( - tree_output: TreeOutput, - universe_decisions: dict[str, Any], -) -> dict[str, Any]: - """Return the decisions visible to a given output for code_version - computation. - - v0.0.7: ``Output.decisions`` is the explicit provenance contract — - the set of decisions whose option choices can change this output. - The Snakefile generator hashes only those into ``code_version``; - we mirror that scoping here so ``lc status`` stays in sync. - Outputs that do not declare decisions hash an empty dict. - """ - declared = tree_output.output_def.get("decisions") or [] - if not declared: - return {} - scoped: dict[str, Any] = {} - prefix = f"{tree_output.analysis_id}." if tree_output.analysis_id else "" - for dec_id in declared: - if prefix and (qualified := f"{prefix}{dec_id}") in universe_decisions: - scoped[dec_id] = universe_decisions[qualified] - elif dec_id in universe_decisions: - scoped[dec_id] = universe_decisions[dec_id] - return scoped - - -def _load_universe_decisions( - project_path: Path, - spec: dict[str, Any], - universe_id: str, -) -> dict[str, Any]: - """Load merged universe decisions if the file exists; empty dict otherwise. - - Universe files are optional during interactive work, so we tolerate - their absence rather than erroring. - """ - universe_yaml = project_path / "universes" / f"{universe_id}.yaml" - if not universe_yaml.exists(): - return {} - try: - return resolve_universe_decisions(project_path, spec, universe_id) - except (FileNotFoundError, KeyError): - return {} - - def get_output_status( project_path: Path, *, universe_id: str, + env: EnvironmentSpec | None = None, + spec: dict[str, Any] | None = None, ) -> Iterator[OutputStatus]: - """Yield an :class:`OutputStatus` for every declared output in the project.""" - spec_path = project_path / "astra.yaml" - spec = resolve_analysis_tree(load_yaml(spec_path), project_path) - universe_decisions = _load_universe_decisions(project_path, spec, universe_id) - project_name = (spec.get("name") or project_path.name).lower().replace(" ", "-") - # Resolve image identities exactly as `lc run` would right now — - # on a kubernetes deployment that's the registry ref, not the local - # tag, or every freshly materialized output would read as stale. - registry = runtime_registry(load_runtime(project_path=project_path).runtime) - resolve_image = make_image_tag_resolver(project_path, project_name, registry=registry) + """Yield an :class:`OutputStatus` for every declared output in the project. + + The recomputed ``code_version`` mirrors the Snakefile generator's + exactly — both call the one shared + :func:`lightcone.engine.manifest.code_version` (via the shared + decision scoping in :mod:`lightcone.engine.tree`), so they can + never disagree. *env* and *spec* are loaded when omitted; callers + iterating several universes pass them through to avoid re-parsing + per universe. + """ + if env is None: + env = load_environment(project_path) + if spec is None: + spec = resolve_analysis_tree( + load_yaml(project_path / "astra.yaml"), project_path + ) + universe_decisions = load_universe_decisions(project_path, spec, universe_id) for tree_out in collect_tree_outputs(spec): out_dir = resolve_output_path(project_path, tree_out, universe_id) / tree_out.output_id @@ -127,44 +89,58 @@ def get_output_status( manifest = read_manifest(out_dir) if manifest is None: - yield OutputStatus( - output_id=tree_out.output_id, - universe_id=universe_id, - analysis_id=tree_out.analysis_id, - output_dir=out_dir, - status="missing", - manifest=None, - recipe_command=recipe_command, - ) - continue - - # Mirror the snakefile generator's image-tag resolution so the - # recomputed code_version matches what was written into the - # manifest at run time. - image_tag = resolve_image(resolve_container_spec(tree_out, spec)) - current_cv = code_version( - recipe=recipe_command, - container_image=image_tag, - decisions=_decisions_for(tree_out, universe_decisions), - ) - if manifest.get("code_version") != current_cv: - yield OutputStatus( - output_id=tree_out.output_id, - universe_id=universe_id, - analysis_id=tree_out.analysis_id, - output_dir=out_dir, - status="stale", - manifest=manifest, - recipe_command=recipe_command, + status: StatusLiteral = "missing" + elif is_pre_migration(manifest): + # An earlier-schema manifest cannot be compared against the + # current identity formula — surfaced distinctly, treated as + # stale by materialize. + status = "pre_migration" + else: + current_cv = code_version( + recipe=recipe_command, + decisions=scoped_decisions_for_output( + tree_out, universe_decisions + ), + env_version=env.env_version, + writable_project=( + tree_out.qualified_id in env.writable_project_outputs + ), ) - continue + status = "ok" if manifest.get("code_version") == current_cv else "stale" yield OutputStatus( output_id=tree_out.output_id, universe_id=universe_id, analysis_id=tree_out.analysis_id, output_dir=out_dir, - status="ok", + status=status, manifest=manifest, recipe_command=recipe_command, ) + + +def env_blast_radius( + project_path: Path, + *, + universes: list[str], + env: EnvironmentSpec | None = None, +) -> int: + """How many materialized outputs the current environment change stales. + + Counts manifests whose recorded ``env_version`` differs from the + environment's current one — printed as + "environment changed: N materialized outputs are now stale" by + ``lc status`` and the materialize preflight, including at escalation + time (declaring the image table IS an environment edit). + """ + if env is None: + env = load_environment(project_path) + count = 0 + for u in universes: + for s in get_output_status(project_path, universe_id=u, env=env): + if s.manifest is None: + continue + recorded = s.manifest.get("env_version") + if recorded is not None and recorded != env.env_version: + count += 1 + return count diff --git a/src/lightcone/engine/tree.py b/src/lightcone/engine/tree.py index f464f0b3..99d82ffa 100644 --- a/src/lightcone/engine/tree.py +++ b/src/lightcone/engine/tree.py @@ -47,6 +47,15 @@ class TreeOutput: analysis_path: str | None # relative path, e.g. "./analyses/hod_fitting" analysis_spec: dict[str, Any] # the sub-analysis spec dict + @property + def qualified_id(self) -> str: + """``analysis_id.output_id`` for sub-analysis outputs, the bare + ``output_id`` at root — the one spelling of an output's identity + (rule keys, writable-project matching, target resolution).""" + if self.analysis_id is None: + return self.output_id + return f"{self.analysis_id}.{self.output_id}" + def collect_tree_outputs(spec: dict[str, Any]) -> list[TreeOutput]: """Walk the resolved tree and collect all outputs with context. @@ -237,25 +246,6 @@ def resolve_output_path( return project_path / "results" / universe_id -def resolve_container_spec( - tree_output: TreeOutput, - root_spec: dict[str, Any], -) -> str | None: - """Pick the container declaration in priority order: - recipe-level > sub-analysis-level > root-level. - Returns the raw spec string (Containerfile path or registry image - tag), or ``None`` when no container is declared at any level. - """ - recipe = tree_output.output_def.get("recipe") or {} - if "container" in recipe: - return recipe["container"] # type: ignore[no-any-return] - if tree_output.analysis_id is not None: - sub = tree_output.analysis_spec.get("container") - if sub is not None: - return sub # type: ignore[no-any-return] - return root_spec.get("container") - - def find_upstream_output( consumer: TreeOutput, inp_id: str, @@ -362,14 +352,60 @@ def resolve_external_input( return None + +def scoped_decisions_for_output( + tree_output: TreeOutput, + universe_decisions: dict[str, Any], +) -> dict[str, Any]: + """The decisions visible to one output, for ``code_version``. + + ``Output.decisions`` is the explicit provenance contract — the set + of decisions whose option choices can change this output. Both the + Snakefile generator (write path) and the status walker (read path) + call this one function, so their scoping can never disagree. + Outputs that declare no decisions hash an empty dict. + """ + declared = tree_output.output_def.get("decisions") or [] + if not declared: + return {} + scoped: dict[str, Any] = {} + prefix = f"{tree_output.analysis_id}." if tree_output.analysis_id else "" + for dec_id in declared: + if prefix and (qualified := f"{prefix}{dec_id}") in universe_decisions: + scoped[dec_id] = universe_decisions[qualified] + elif dec_id in universe_decisions: + scoped[dec_id] = universe_decisions[dec_id] + return scoped + + +def load_universe_decisions( + project_path: Path, + spec: dict[str, Any], + universe_id: str, +) -> dict[str, Any]: + """Merged universe decisions when the file exists; empty otherwise. + + Universe files are optional during interactive work, so absence is + tolerated rather than an error. + """ + universe_yaml = project_path / "universes" / f"{universe_id}.yaml" + if not universe_yaml.exists(): + return {} + try: + return resolve_universe_decisions(project_path, spec, universe_id) + except (FileNotFoundError, KeyError): + return {} + + __all__ = [ "TreeOutput", "collect_tree_inputs", "collect_tree_outputs", "find_upstream_output", "get_decisions_for_analysis", - "resolve_container_spec", + "load_universe_decisions", "resolve_external_input", "resolve_output_path", "resolve_universe_decisions", + "scoped_decisions_for_output", ] diff --git a/src/lightcone/engine/uv_env.py b/src/lightcone/engine/uv_env.py new file mode 100644 index 00000000..9c75a31c --- /dev/null +++ b/src/lightcone/engine/uv_env.py @@ -0,0 +1,54 @@ +"""The uv environment-variable contract. + +Two closed lists (spec §4, §6): + +* :data:`SCRUB_LIST` — ambient ``UV_*`` variables the launcher unsets + before any uv invocation. Ambient uv steering could silently redirect + the environment lc converges (a different project env, a different + index, a different interpreter). Explicit flags beat ambient + variables in uv (verified, §13), so lc's always-pass-explicit-flags + posture makes the scrub defense-in-depth — but scrubbing keeps the + posture honest even for settings lc has no flag for. + +* :data:`OFFLINE_OVERLAY` — applied to every worker/recipe exec after + convergence: converge once, then never write to the environment. + +Audited against uv 0.12 (the project's pinned engine version). +""" +from __future__ import annotations + +from collections.abc import MutableMapping + +SCRUB_LIST: tuple[str, ...] = ( + "UV_CACHE_DIR", + "UV_CONFIG_FILE", + "UV_EXTRA_INDEX_URL", + "UV_FIND_LINKS", + "UV_FROZEN", + "UV_INDEX_URL", + "UV_LINK_MODE", + "UV_LOCKED", + "UV_NO_CACHE", + "UV_NO_CONFIG", + "UV_NO_SYNC", + "UV_OFFLINE", + "UV_PROJECT", + "UV_PROJECT_ENVIRONMENT", + "UV_PYTHON", + "UV_PYTHON_DOWNLOADS", + "UV_PYTHON_INSTALL_DIR", +) + +#: Converge once, then never write: applied to recipe/worker execs so a +#: mid-run ``uv run`` can neither hit the network nor fetch an +#: interpreter. +OFFLINE_OVERLAY: dict[str, str] = { + "UV_OFFLINE": "1", + "UV_PYTHON_DOWNLOADS": "never", +} + + +def scrub(env: MutableMapping[str, str]) -> None: + """Remove every scrub-listed variable from *env*, in place.""" + for name in SCRUB_LIST: + env.pop(name, None) diff --git a/src/lightcone/engine/verify.py b/src/lightcone/engine/verify.py index 51065300..6e150fa3 100644 --- a/src/lightcone/engine/verify.py +++ b/src/lightcone/engine/verify.py @@ -11,6 +11,12 @@ - ``broken_chain``: the recorded ``input_versions`` reference an upstream output whose own ``data_version`` no longer matches. +Orthogonal to pass/fail, verify surfaces provenance *notes* the hashes +cannot express: ``pre_migration`` (an earlier-schema manifest — its +hashes are still checked), ``dirty_tree`` (materialized from an +uncommitted working tree), and ``unsandboxed`` (no enforcement +mechanism ran). + Like ``status``, this module never imports Snakemake. """ from __future__ import annotations @@ -18,11 +24,11 @@ from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Any, Literal from astra.helpers import load_yaml, resolve_analysis_tree -from lightcone.engine.manifest import read_manifest, sha256_dir +from lightcone.engine.manifest import is_pre_migration, read_manifest, sha256_dir from lightcone.engine.tree import ( collect_tree_outputs, find_upstream_output, @@ -40,15 +46,36 @@ class VerifyResult: passed: bool failure: FailureKind | None detail: str | None = None + notes: tuple[str, ...] = () + + +def _manifest_notes(manifest: dict[str, Any]) -> tuple[str, ...]: + notes: list[str] = [] + if is_pre_migration(manifest): + notes.append("pre_migration") + if manifest.get("git_dirty"): + notes.append("dirty_tree") + hermeticity = manifest.get("hermeticity") + if not hermeticity or hermeticity.get("mechanism") in (None, "none"): + notes.append("unsandboxed") + return tuple(notes) def verify_outputs( project_path: Path, *, universe_id: str, + spec: dict[str, Any] | None = None, ) -> Iterator[VerifyResult]: - """Yield a :class:`VerifyResult` for every output with a recipe.""" - spec = resolve_analysis_tree(load_yaml(project_path / "astra.yaml"), project_path) + """Yield a :class:`VerifyResult` for every output with a recipe. + + *spec* (the resolved analysis tree) is loaded when omitted; callers + iterating several universes pass it through. + """ + if spec is None: + spec = resolve_analysis_tree( + load_yaml(project_path / "astra.yaml"), project_path + ) all_outputs = collect_tree_outputs(spec) for tree_out in all_outputs: @@ -74,6 +101,8 @@ def verify_outputs( ) continue + notes = _manifest_notes(manifest) + actual_dv = sha256_dir(out_dir) if actual_dv != manifest.get("data_version"): yield VerifyResult( @@ -86,6 +115,7 @@ def verify_outputs( f"recorded {manifest.get('data_version')!r} != " f"actual {actual_dv!r}" ), + notes=notes, ) continue @@ -126,6 +156,7 @@ def verify_outputs( passed=False, failure="broken_chain", detail=chain_failure, + notes=notes, ) continue @@ -135,6 +166,7 @@ def verify_outputs( output_dir=out_dir, passed=True, failure=None, + notes=notes, ) diff --git a/src/lightcone/engine/wrroc.py b/src/lightcone/engine/wrroc.py index af9f72f1..a0026068 100644 --- a/src/lightcone/engine/wrroc.py +++ b/src/lightcone/engine/wrroc.py @@ -454,7 +454,7 @@ def _add_create_action( instrument_id = self._add_recipe_software( recipe_cmd, - manifest.get("container_image"), + (manifest.get("image") or {}).get("tag"), tool_name=(tree_out.output_def.get("recipe") or {}).get("tool_name"), output_id=tree_out.output_id, ) diff --git a/src/lightcone/launcher.py b/src/lightcone/launcher.py new file mode 100644 index 00000000..6613e79d --- /dev/null +++ b/src/lightcone/launcher.py @@ -0,0 +1,155 @@ +"""The tool-env launcher: discover → mode-detect → scrub → converge → exec. + +``lc`` is installed as a uv tool (or any ambient environment); the +*engine* that executes a project lives inside the project's own lock +("the engine is inside the experiment's lock"). The launcher bridges +the two: for verbs that operate on the recipe environment it converges +that environment and re-execs the project's own ``lc`` binary from it. + +The delegation interface is **minimal and frozen**: argv passthrough +plus ``LC_DELEGATED=1``. A tool-env launcher of any version must be +able to delegate to a project-locked engine of any age — nothing else +may ever travel across this boundary. + +Verbs in :data:`TOOL_ENV_VERBS` run directly in the tool env: they are +pre-lock (``init``), offline/manifest-driven (``status``, ``verify``, +``export``), or build the environment itself (``build`` — no delegable +environment exists before the image does). +""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +from lightcone.engine import uv_env +from lightcone.engine.contract import DELEGATED_ENV +from lightcone.engine.environment import ( + EnvironmentSpec, + Mode, + ProjectEnvironmentError, + load_environment, +) +from lightcone.engine.project import find_root + +#: Verbs that never delegate. +TOOL_ENV_VERBS = frozenset({"init", "status", "verify", "build", "export"}) + + +def _fail(message: str) -> None: + sys.stderr.write(f"Error: {message}\n") + raise SystemExit(1) + + +def maybe_delegate(argv: list[str]) -> None: + """Delegate to the project-locked engine when appropriate. + + Either returns (caller proceeds with normal Click dispatch in this + environment) or execs the project engine and never returns. + """ + if os.environ.get(DELEGATED_ENV) == "1": + return + verb = next((a for a in argv if not a.startswith("-")), None) + if verb is None or verb in TOOL_ENV_VERBS: + return + + root = find_root() + if root is None: + return # Click renders the no-project error uniformly. + + try: + env = load_environment(root) + except ProjectEnvironmentError as e: + _fail(str(e)) + return # unreachable; keeps mypy honest + + # Ambient UV_* steering could redirect which environment converges; + # lc always passes explicit flags, and the scrub closes the rest. + uv_env.scrub(os.environ) + + if env.mode is Mode.CONTAINERIZED: + _delegate_containerized(root, env, argv, verb) + return # unreachable (exec or SystemExit) + + _converge_direct(root) + + engine = env.venv / "bin" / "lc" + if not engine.is_file(): + _fail( + f"{engine} does not exist after a successful sync — the " + "project's lock must include lightcone-cli (the engine lives " + "inside the experiment's lock): `uv add lightcone-cli`." + ) + # Direct exec — the frozen delegation interface: argv passthrough + + # LC_DELEGATED=1. Never a PATH fallback. + os.execve( + str(engine), + ["lc", *argv], + {**os.environ, DELEGATED_ENV: "1"}, + ) + + +def _delegate_containerized( + root: Path, env: EnvironmentSpec, argv: list[str], verb: str +) -> None: + """Full-stack delegation: one ``podman run`` hosts the delegated + engine, its dask workers, the child snakemake, and every recipe — + all from the image's baked ``/opt/venv``. There is no host ``.venv`` + for the project at all. + + ``lc materialize`` builds a missing image (announced); ``lc run`` + never builds — it errors with the exact ``lc build`` command. + """ + from lightcone.engine.image import ( + ImageError, + ensure_image, + resolve_pinned, + ) + from lightcone.engine.image.machine import machine_preflight + from lightcone.engine.image.mounts import ( + compute_mount_set, + external_input_paths, + ) + from lightcone.engine.image.runtime_podman import PodmanRuntime + + try: + runtime = PodmanRuntime() + if verb == "materialize": + record = ensure_image( + root, + env, + on_progress=lambda msg: print(msg, file=sys.stderr), + ) + else: + record = resolve_pinned(root, env) + mounts = compute_mount_set( + root, + external_inputs=external_input_paths(root), + readonly_project=(verb == "run"), + ) + machine_preflight(mounts.sources()) + except ImageError as e: + _fail(str(e)) + return # unreachable + + runtime.exec_full_stack(record=record, mounts=mounts, lc_argv=argv) + + +def _converge_direct(root: Path) -> None: + """``uv sync --locked --exact`` — converge once; workers then run + with the offline overlay and never write to the environment.""" + proc = subprocess.run( + [ + "uv", "sync", "--locked", "--exact", "--compile-bytecode", + "--project", str(root), + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + _fail( + "`uv sync --locked --exact` failed — the lock and " + f"pyproject.toml disagree, or uv is unavailable:\n{proc.stderr.strip()}" + ) diff --git a/src/snakemake_executor_plugin_dask/__init__.py b/src/snakemake_executor_plugin_dask/__init__.py index 3cff3f63..8fb8b6f8 100644 --- a/src/snakemake_executor_plugin_dask/__init__.py +++ b/src/snakemake_executor_plugin_dask/__init__.py @@ -1,19 +1,13 @@ """Snakemake executor plugin: dispatches each rule's shell command to a running ``dask.distributed`` cluster. -The cluster rendezvous is read from the environment: either a plain -scheduler address in ``DASK_SCHEDULER_ADDRESS`` or a Dask Gateway -cluster name in ``LIGHTCONE_GATEWAY_CLUSTER`` (rejoined through the -Gateway API — ``gateway://`` schedulers cannot be dialled directly). -``lc run`` is responsible for setting one of them — typically by -constructing a ``LocalCluster()`` for the duration of the run, backed by -``srun``-launched workers inside a SLURM allocation, or by creating a -run-scoped Gateway cluster on a JupyterHub deployment. +The cluster rendezvous is read from the environment: a scheduler address +in ``DASK_SCHEDULER_ADDRESS``, set by ``lc materialize`` when it +constructs the run-scoped ``LocalCluster``. The plugin is intentionally minimal: each Snakemake job becomes a -``client.submit(_run_shell, cmd, resources={...})`` call. Workers run the -shell command as-is (recipes are already containerized at Snakefile -generation time, so the worker just shells out). +``client.submit(_run_shell, cmd, resources={...})`` call. Workers run +the shell command as-is. """ from snakemake_interface_executor_plugins.settings import ( # type: ignore[import-untyped] diff --git a/src/snakemake_executor_plugin_dask/executor.py b/src/snakemake_executor_plugin_dask/executor.py index 03d2a52a..bb15a6ac 100644 --- a/src/snakemake_executor_plugin_dask/executor.py +++ b/src/snakemake_executor_plugin_dask/executor.py @@ -19,7 +19,6 @@ ) from lightcone.engine.dask_cluster import ( - GATEWAY_CLUSTER_ENV, RESOURCE_CPUS, RESOURCE_GPUS, RESOURCE_MEMORY, @@ -70,16 +69,6 @@ def _run_shell(cmd: str) -> tuple[int, str]: return p.returncode, block -def _unpack_result(result: object) -> tuple[int, str]: - """Accept both the current ``(exit_code, block)`` result and the - bare ``int`` a worker running an older lightcone-cli release returns - (dask resolves ``_run_shell`` by module path on the worker, so - driver and worker versions can skew on image-based deployments).""" - if isinstance(result, tuple) and len(result) == 2: - return int(result[0]), str(result[1]) - return int(result), "" # type: ignore[call-overload] - - def _build_resources(job: JobExecutorInterface) -> dict[str, float]: """Translate Snakemake resources to Dask abstract resource units.""" res: dict[str, float] = {} @@ -96,41 +85,18 @@ def _build_resources(job: JobExecutorInterface) -> dict[str, float]: def _connect_client(): # type: ignore[no-untyped-def] - """Connect to the run's cluster. - - Two rendezvous modes, both set up by ``lc run``: - - - :data:`GATEWAY_CLUSTER_ENV` names a Dask Gateway cluster the - parent created. Gateway schedulers speak a ``gateway://`` comm - scheme with per-cluster TLS credentials held by the Gateway API — - a bare ``Client`` cannot dial them, so we rejoin through - ``Gateway().connect(name)``. - - Otherwise ``DASK_SCHEDULER_ADDRESS`` is a plain scheduler address. + """Connect to the run's cluster via ``DASK_SCHEDULER_ADDRESS``. - Returns ``(client, closer)`` where *closer* releases everything the - rendezvous opened. + The address is set up by ``lc materialize`` (the run-scoped + LocalCluster's scheduler). Returns ``(client, closer)`` where + *closer* releases everything the rendezvous opened. """ from dask.distributed import Client - if name := os.environ.get(GATEWAY_CLUSTER_ENV): - from dask_gateway import Gateway - - # shutdown_on_close=False: the parent lc run owns the cluster - # lifecycle; the executor is a guest. - cluster = Gateway().connect(name, shutdown_on_close=False) - client = cluster.get_client() - - def closer() -> None: - client.close() - cluster.close() - - return client, closer - addr = os.environ.get("DASK_SCHEDULER_ADDRESS") if not addr: raise WorkflowError( - "Neither DASK_SCHEDULER_ADDRESS nor " - f"{GATEWAY_CLUSTER_ENV} is set. `lc run` should set one " + "DASK_SCHEDULER_ADDRESS is not set. `lc materialize` sets it " "before invoking snakemake; if you're calling snakemake " "directly, point it at a running dask scheduler." ) @@ -193,7 +159,7 @@ async def check_active_jobs( ) continue - exit_code, block = _unpack_result(future.result()) + exit_code, block = future.result() if block: # One atomic write per finished rule. We run inside the # parent snakemake process, so this is naturally diff --git a/tests/conftest.py b/tests/conftest.py index baba0659..f379683a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,81 @@ -"""Shared test fixtures for lightcone-cli tests.""" - +"""Shared fixtures for the lightcone-cli test suite.""" from __future__ import annotations + +from pathlib import Path + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--regen-goldens", + action="store_true", + default=False, + help="Rewrite golden fixture files instead of asserting against them.", + ) + +#: Deterministic environment-file contents shared by fixture projects — +#: identity tests pin hashes over these exact bytes. +PYPROJECT_MIN = """\ +[project] +name = "fixture-proj" +version = "0.0.0" +requires-python = ">=3.12" +dependencies = [] +""" + +UV_LOCK_MIN = """\ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "fixture-proj" +version = "0.0.0" +source = { virtual = "." } +""" + +PYTHON_VERSION_MIN = "3.12.12\n" + +ASTRA_YAML_MIN = """\ +outputs: + - id: result + type: metric + recipe: + command: echo hi > {output}/data.txt +""" + +IMAGE_TABLE = """ +[tool.lightcone.image] +system-packages = ["r-base-core", "libhdf5-dev"] +""" + + +def make_project( + root: Path, + *, + containerized: bool = False, + extra_pyproject: str = "", + astra_yaml: str = ASTRA_YAML_MIN, +) -> Path: + """Write a minimal, deterministic uv project scaffold under *root*.""" + root.mkdir(parents=True, exist_ok=True) + pyproject = PYPROJECT_MIN + if containerized: + pyproject += IMAGE_TABLE + pyproject += extra_pyproject + (root / "pyproject.toml").write_text(pyproject) + (root / "uv.lock").write_text(UV_LOCK_MIN) + (root / ".python-version").write_text(PYTHON_VERSION_MIN) + (root / "astra.yaml").write_text(astra_yaml) + return root + + +@pytest.fixture +def direct_project(tmp_path: Path) -> Path: + return make_project(tmp_path / "proj") + + +@pytest.fixture +def containerized_project(tmp_path: Path) -> Path: + return make_project(tmp_path / "proj", containerized=True) diff --git a/tests/goldens/custom-base.Containerfile b/tests/goldens/custom-base.Containerfile new file mode 100644 index 00000000..c94101c2 --- /dev/null +++ b/tests/goldens/custom-base.Containerfile @@ -0,0 +1,45 @@ +# generated by lightcone-cli — DO NOT EDIT +# (rendered from pyproject.toml [tool.lightcone.image]; regenerated by `lc build`) +FROM nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f AS env +SHELL ["/bin/sh", "-c"] + +# base contract checks — each failure is a distinct exit code the +# builder maps to a pointed error (never a raw build log) +RUN test -x /bin/sh || exit 41 +RUN if ls /lib/ld-musl-* >/dev/null 2>&1; then echo 'musl base' >&2; exit 43; fi +RUN command -v apt-get >/dev/null 2>&1 || exit 44 + +# system layer — installed BEFORE uv sync so lock-level system +# dependencies (sdist builds, rpy2-style imports) resolve here +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + texlive-latex-base \ + && rm -rf /var/lib/apt/lists/* + +# pinned uv binary (manifest-list digest — arch-independent text) +COPY --from=ghcr.io/astral-sh/uv@sha256:2d890623d310b57771ce840f0da5eed5fc6d657da05ffaa45d82797b53fa3abc /uv /opt/lc/bin/uv + +# exact interpreter, uv-managed, outside the base's control +ENV UV_PYTHON_INSTALL_DIR=/opt/python +RUN /opt/lc/bin/uv python install 3.12.12 + +# locked sync — code-free: pyproject.toml + uv.lock are the only +# project files that ever enter an image (G5) +WORKDIR /opt/lc/project +COPY pyproject.toml uv.lock ./ +ENV UV_PROJECT_ENVIRONMENT=/opt/venv +RUN /opt/lc/bin/uv sync --locked --exact --no-install-project --compile-bytecode --python 3.12.12 --project /opt/lc/project + +FROM env AS final +LABEL io.lightcone.env-version="sha256:abababababababababababababababababababababababababababababababab" +# attestation: name-pinned apt layer's actual package versions +RUN dpkg -l > /opt/lc/dpkg-snapshot.txt 2>/dev/null || echo 'dpkg unavailable' > /opt/lc/dpkg-snapshot.txt +RUN printf '%s' '{"env_version":"sha256:abababababababababababababababababababababababababababababababab","python_version":"3.12.12","uv_version":"0.12.3"}' > /opt/lc/identity.json +# world-readable: the invoking uid (rootless --userns=keep-id) +# must be able to read everything lc baked +RUN chmod -R a+rX /opt/lc /opt/python /opt/venv +# offline overlay — FINAL stage only; earlier stages keep network +ENV UV_OFFLINE=1 UV_PYTHON_DOWNLOADS=never UV_NO_SYNC=1 \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + UV_PYTHON_INSTALL_DIR=/opt/python \ + PATH=/opt/venv/bin:/opt/lc/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/tests/goldens/extra-stage.Containerfile b/tests/goldens/extra-stage.Containerfile new file mode 100644 index 00000000..86880b24 --- /dev/null +++ b/tests/goldens/extra-stage.Containerfile @@ -0,0 +1,49 @@ +# generated by lightcone-cli — DO NOT EDIT +# (rendered from pyproject.toml [tool.lightcone.image]; regenerated by `lc build`) +FROM docker.io/library/debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS env +SHELL ["/bin/sh", "-c"] + +# base contract checks — each failure is a distinct exit code the +# builder maps to a pointed error (never a raw build log) +RUN test -x /bin/sh || exit 41 +RUN if ls /lib/ld-musl-* >/dev/null 2>&1; then echo 'musl base' >&2; exit 43; fi +RUN command -v apt-get >/dev/null 2>&1 || exit 44 + +# system layer — installed BEFORE uv sync so lock-level system +# dependencies (sdist builds, rpy2-style imports) resolve here +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + r-base-core \ + && rm -rf /var/lib/apt/lists/* + +# pinned uv binary (manifest-list digest — arch-independent text) +COPY --from=ghcr.io/astral-sh/uv@sha256:2d890623d310b57771ce840f0da5eed5fc6d657da05ffaa45d82797b53fa3abc /uv /opt/lc/bin/uv + +# exact interpreter, uv-managed, outside the base's control +ENV UV_PYTHON_INSTALL_DIR=/opt/python +RUN /opt/lc/bin/uv python install 3.12.12 + +# locked sync — code-free: pyproject.toml + uv.lock are the only +# project files that ever enter an image (G5) +WORKDIR /opt/lc/project +COPY pyproject.toml uv.lock ./ +ENV UV_PROJECT_ENVIRONMENT=/opt/venv +RUN /opt/lc/bin/uv sync --locked --exact --no-install-project --compile-bytecode --python 3.12.12 --project /opt/lc/project + +# user extra stage (Containerfile.extra, verbatim) +FROM env AS extra +RUN Rscript -e 'install.packages("cmdstanr")' + +FROM extra AS final +LABEL io.lightcone.env-version="sha256:abababababababababababababababababababababababababababababababab" +# attestation: name-pinned apt layer's actual package versions +RUN dpkg -l > /opt/lc/dpkg-snapshot.txt 2>/dev/null || echo 'dpkg unavailable' > /opt/lc/dpkg-snapshot.txt +RUN printf '%s' '{"env_version":"sha256:abababababababababababababababababababababababababababababababab","python_version":"3.12.12","uv_version":"0.12.3"}' > /opt/lc/identity.json +# world-readable: the invoking uid (rootless --userns=keep-id) +# must be able to read everything lc baked +RUN chmod -R a+rX /opt/lc /opt/python /opt/venv +# offline overlay — FINAL stage only; earlier stages keep network +ENV UV_OFFLINE=1 UV_PYTHON_DOWNLOADS=never UV_NO_SYNC=1 \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + UV_PYTHON_INSTALL_DIR=/opt/python \ + PATH=/opt/venv/bin:/opt/lc/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/tests/goldens/minimal.Containerfile b/tests/goldens/minimal.Containerfile new file mode 100644 index 00000000..df1fd9d2 --- /dev/null +++ b/tests/goldens/minimal.Containerfile @@ -0,0 +1,37 @@ +# generated by lightcone-cli — DO NOT EDIT +# (rendered from pyproject.toml [tool.lightcone.image]; regenerated by `lc build`) +FROM docker.io/library/debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS env +SHELL ["/bin/sh", "-c"] + +# base contract checks — each failure is a distinct exit code the +# builder maps to a pointed error (never a raw build log) +RUN test -x /bin/sh || exit 41 +RUN if ls /lib/ld-musl-* >/dev/null 2>&1; then echo 'musl base' >&2; exit 43; fi + +# pinned uv binary (manifest-list digest — arch-independent text) +COPY --from=ghcr.io/astral-sh/uv@sha256:2d890623d310b57771ce840f0da5eed5fc6d657da05ffaa45d82797b53fa3abc /uv /opt/lc/bin/uv + +# exact interpreter, uv-managed, outside the base's control +ENV UV_PYTHON_INSTALL_DIR=/opt/python +RUN /opt/lc/bin/uv python install 3.12.12 + +# locked sync — code-free: pyproject.toml + uv.lock are the only +# project files that ever enter an image (G5) +WORKDIR /opt/lc/project +COPY pyproject.toml uv.lock ./ +ENV UV_PROJECT_ENVIRONMENT=/opt/venv +RUN /opt/lc/bin/uv sync --locked --exact --no-install-project --compile-bytecode --python 3.12.12 --project /opt/lc/project + +FROM env AS final +LABEL io.lightcone.env-version="sha256:abababababababababababababababababababababababababababababababab" +# attestation: name-pinned apt layer's actual package versions +RUN dpkg -l > /opt/lc/dpkg-snapshot.txt 2>/dev/null || echo 'dpkg unavailable' > /opt/lc/dpkg-snapshot.txt +RUN printf '%s' '{"env_version":"sha256:abababababababababababababababababababababababababababababababab","python_version":"3.12.12","uv_version":"0.12.3"}' > /opt/lc/identity.json +# world-readable: the invoking uid (rootless --userns=keep-id) +# must be able to read everything lc baked +RUN chmod -R a+rX /opt/lc /opt/python /opt/venv +# offline overlay — FINAL stage only; earlier stages keep network +ENV UV_OFFLINE=1 UV_PYTHON_DOWNLOADS=never UV_NO_SYNC=1 \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + UV_PYTHON_INSTALL_DIR=/opt/python \ + PATH=/opt/venv/bin:/opt/lc/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/tests/goldens/packages.Containerfile b/tests/goldens/packages.Containerfile new file mode 100644 index 00000000..84b83863 --- /dev/null +++ b/tests/goldens/packages.Containerfile @@ -0,0 +1,45 @@ +# generated by lightcone-cli — DO NOT EDIT +# (rendered from pyproject.toml [tool.lightcone.image]; regenerated by `lc build`) +FROM docker.io/library/debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS env +SHELL ["/bin/sh", "-c"] + +# base contract checks — each failure is a distinct exit code the +# builder maps to a pointed error (never a raw build log) +RUN test -x /bin/sh || exit 41 +RUN if ls /lib/ld-musl-* >/dev/null 2>&1; then echo 'musl base' >&2; exit 43; fi +RUN command -v apt-get >/dev/null 2>&1 || exit 44 + +# system layer — installed BEFORE uv sync so lock-level system +# dependencies (sdist builds, rpy2-style imports) resolve here +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libhdf5-dev r-base-core \ + && rm -rf /var/lib/apt/lists/* + +# pinned uv binary (manifest-list digest — arch-independent text) +COPY --from=ghcr.io/astral-sh/uv@sha256:2d890623d310b57771ce840f0da5eed5fc6d657da05ffaa45d82797b53fa3abc /uv /opt/lc/bin/uv + +# exact interpreter, uv-managed, outside the base's control +ENV UV_PYTHON_INSTALL_DIR=/opt/python +RUN /opt/lc/bin/uv python install 3.12.12 + +# locked sync — code-free: pyproject.toml + uv.lock are the only +# project files that ever enter an image (G5) +WORKDIR /opt/lc/project +COPY pyproject.toml uv.lock ./ +ENV UV_PROJECT_ENVIRONMENT=/opt/venv +RUN /opt/lc/bin/uv sync --locked --exact --no-install-project --compile-bytecode --python 3.12.12 --project /opt/lc/project + +FROM env AS final +LABEL io.lightcone.env-version="sha256:abababababababababababababababababababababababababababababababab" +# attestation: name-pinned apt layer's actual package versions +RUN dpkg -l > /opt/lc/dpkg-snapshot.txt 2>/dev/null || echo 'dpkg unavailable' > /opt/lc/dpkg-snapshot.txt +RUN printf '%s' '{"env_version":"sha256:abababababababababababababababababababababababababababababababab","python_version":"3.12.12","uv_version":"0.12.3"}' > /opt/lc/identity.json +# world-readable: the invoking uid (rootless --userns=keep-id) +# must be able to read everything lc baked +RUN chmod -R a+rX /opt/lc /opt/python /opt/venv +# offline overlay — FINAL stage only; earlier stages keep network +ENV UV_OFFLINE=1 UV_PYTHON_DOWNLOADS=never UV_NO_SYNC=1 \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + UV_PYTHON_INSTALL_DIR=/opt/python \ + PATH=/opt/venv/bin:/opt/lc/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/tests/test_cli.py b/tests/test_cli.py index 4bec6685..5fc210b8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,14 @@ -"""Tests for the redesigned lightcone CLI.""" +"""Tests for the lightcone CLI surface.""" from __future__ import annotations import json -import shutil import subprocess from pathlib import Path from unittest.mock import MagicMock import pytest from click.testing import CliRunner +from conftest import PYPROJECT_MIN, PYTHON_VERSION_MIN, UV_LOCK_MIN from lightcone.cli.commands import main @@ -20,21 +20,49 @@ def runner() -> CliRunner: @pytest.fixture(autouse=True) def _isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Redirect ``~/.lightcone/`` to a temp dir so tests don't pollute the user's - real config. The global config is auto-created on first ``lc`` invocation.""" + """Redirect ``~`` to a temp dir so tests can't touch the user's real + home.""" fake_home = tmp_path / "_home" fake_home.mkdir() monkeypatch.setattr(Path, "home", lambda: fake_home) return fake_home +@pytest.fixture(autouse=True) +def _fake_uv(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + """Fake the uv seam so init tests are hermetic (no network, no real + resolution). Emulates the observable effects: ``uv lock`` writes + uv.lock, ``uv sync`` materializes .venv.""" + calls: list[list[str]] = [] + + def fake_run_uv(args: list[str], *, cwd: Path) -> MagicMock: + calls.append(list(args)) + if args[0] == "lock": + project = Path(args[args.index("--project") + 1]) + (project / "uv.lock").write_text(UV_LOCK_MIN) + elif args[0] == "sync": + project = Path(args[args.index("--project") + 1]) + (project / ".venv" / "bin").mkdir(parents=True, exist_ok=True) + return MagicMock(returncode=0, stdout="", stderr="") + + from lightcone.cli import commands + + monkeypatch.setattr(commands, "_run_uv", fake_run_uv) + # NB: commands.shutil IS the global shutil module — accept the + # path= kwarg the sandbox policy's which() calls use. + monkeypatch.setattr( + commands.shutil, "which", lambda name, path=None: f"/usr/bin/{name}" + ) + return calls + + # ---- top-level ------------------------------------------------------------ def test_help_lists_core_commands(runner: CliRunner) -> None: result = runner.invoke(main, ["--help"]) assert result.exit_code == 0 - for cmd in ("init", "run", "status", "verify", "build"): + for cmd in ("init", "materialize", "run", "status", "verify", "build"): assert cmd in result.output @@ -45,20 +73,23 @@ def test_help_does_not_advertise_removed_commands(runner: CliRunner) -> None: assert " setup " not in result.output -def test_first_invocation_auto_creates_global_config( - runner: CliRunner, _isolated_home: Path, tmp_path: Path +def test_engine_errors_render_cleanly( + runner: CliRunner, monkeypatch: pytest.MonkeyPatch ) -> None: - config = _isolated_home / ".lightcone" / "config.yaml" - assert not config.exists() - # Any real subcommand triggers the group callback; ``init`` runs cleanly - # without a pre-existing project. - project = tmp_path / "proj" - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv"] - ) - assert result.exit_code == 0, result.output - assert config.exists() - assert "runtime: auto" in config.read_text() + """ProjectEnvironmentError from any command surfaces as a one-line + CLI error (exit 1), not a traceback — the group boundary translates + it.""" + from lightcone.cli import commands + from lightcone.engine.environment import ProjectEnvironmentError + + def _boom(*args: object, **kwargs: object) -> Path: + raise ProjectEnvironmentError("no uv.lock — run `uv lock`") + + monkeypatch.setattr(commands, "_project_root", _boom) + result = runner.invoke(main, ["status"]) + assert result.exit_code == 1 + assert "uv.lock" in result.output + assert "Traceback" not in result.output # ---- lc init -------------------------------------------------------------- @@ -66,38 +97,109 @@ def test_first_invocation_auto_creates_global_config( def test_init_creates_project(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 0, result.output assert (project / "astra.yaml").exists() + assert (project / "pyproject.toml").exists() + assert (project / ".python-version").exists() + assert (project / "uv.lock").exists() + assert (project / ".venv").is_dir() + assert (project / "AGENTS.md").exists() assert (project / ".gitignore").exists() assert (project / ".lightcone").is_dir() assert (project / "results").is_dir() assert (project / "universes").is_dir() # The README is the durable hint that outputs materialize here via - # lc run — and the one file in results/ that stays tracked by git. + # lc materialize — and the one file in results/ that stays tracked by git. readme = (project / "results" / "README.md").read_text() - assert "lc run" in readme + assert "lc materialize" in readme gitignore = (project / ".gitignore").read_text() assert "results/*" in gitignore assert "!results/README.md" in gitignore + assert ".venv/" in gitignore + assert ".lightcone/image/" in gitignore + + +def test_init_uv_scaffold_content(runner: CliRunner, tmp_path: Path) -> None: + """The scaffolded uv project: virtual (no build-system), the engine + inside the experiment's lock, an exact interpreter pin.""" + project = tmp_path / "proj" + result = runner.invoke(main, ["init", str(project), "--no-git"]) + assert result.exit_code == 0, result.output + + pyproject = (project / "pyproject.toml").read_text() + assert "lightcone-cli" in pyproject + assert "[build-system]" not in pyproject + assert "[tool.uv]" in pyproject + + pin = (project / ".python-version").read_text().strip() + assert pin.count(".") == 2 # exact patch, e.g. 3.12.12 + + agents = (project / "AGENTS.md").read_text() + assert "uv add" in agents + assert "lc materialize" in agents + + +def test_init_no_containerfile_scaffolded(runner: CliRunner, tmp_path: Path) -> None: + """v6: images are generated from the lock — no authored Containerfile, + no requirements.txt.""" + project = tmp_path / "proj" + result = runner.invoke(main, ["init", str(project), "--no-git"]) + assert result.exit_code == 0, result.output + assert not (project / "Containerfile").exists() + assert not (project / "requirements.txt").exists() + # And the astra boilerplate's container: line is stripped. + assert "container:" not in (project / "astra.yaml").read_text() + + +def test_init_refuses_authored_containerfile( + runner: CliRunner, tmp_path: Path +) -> None: + """The user's own file operation is the consent to migrate — init + refuses with instructions, even under --check.""" + project = tmp_path / "proj" + project.mkdir() + (project / "Containerfile").write_text("FROM python:3.12-slim\n") + for extra in ([], ["--check"]): + result = runner.invoke(main, ["init", str(project), "--no-git", *extra]) + assert result.exit_code != 0 + assert "delete or rename" in result.output + assert (project / "Containerfile").read_text() == "FROM python:3.12-slim\n" + + +def test_init_invokes_uv_lock_and_sync( + runner: CliRunner, tmp_path: Path, _fake_uv: list[list[str]] +) -> None: + project = tmp_path / "proj" + result = runner.invoke(main, ["init", str(project), "--no-git"]) + assert result.exit_code == 0, result.output + assert ["lock", "--project", str(project)] in _fake_uv + assert [ + "sync", "--locked", "--exact", "--compile-bytecode", + "--project", str(project), + ] in _fake_uv -def test_init_creates_report_template(runner: CliRunner, tmp_path: Path) -> None: +def test_init_no_sync_skips_venv( + runner: CliRunner, tmp_path: Path, _fake_uv: list[list[str]] +) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) + result = runner.invoke(main, ["init", str(project), "--no-git", "--no-sync"]) assert result.exit_code == 0, result.output + assert (project / "uv.lock").exists() + assert not (project / ".venv").exists() + assert not any(args[0] == "sync" for args in _fake_uv) - myst_yml = (project / "myst.yml").read_text() - assert "mystra.mjs" in myst_yml - assert "index.md" in myst_yml - index_md = (project / "index.md").read_text() - assert index_md.startswith("# proj\n") - # References must track the astra init boilerplate element ids. - assert "{astra}`decisions.example_method`" in index_md - assert "{astra:value}`outputs.main_result`" in index_md +def test_init_requires_uv( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from lightcone.cli import commands - assert "_build/" in (project / ".gitignore").read_text() + monkeypatch.setattr(commands.shutil, "which", lambda _: None) + result = runner.invoke(main, ["init", str(tmp_path / "proj"), "--no-git"]) + assert result.exit_code != 0 + assert "uv is required" in result.output def test_init_adopts_existing_project(runner: CliRunner, tmp_path: Path) -> None: @@ -107,28 +209,42 @@ def test_init_adopts_existing_project(runner: CliRunner, tmp_path: Path) -> None project.mkdir() (project / "astra.yaml").write_text("# user spec\n") (project / ".gitignore").write_text("*.log\n") - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) + (project / "pyproject.toml").write_text(PYPROJECT_MIN) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 0, result.output # User files untouched (gitignore gains the managed block, keeps content). assert (project / "astra.yaml").read_text() == "# user spec\n" + assert (project / "pyproject.toml").read_text() == PYPROJECT_MIN gitignore = (project / ".gitignore").read_text() assert gitignore.startswith("*.log\n") assert "# lightcone-cli" in gitignore # Missing lightcone pieces were created. - assert (project / "Containerfile").exists() assert (project / ".lightcone" / "lightcone.yaml").exists() + assert (project / ".python-version").exists() + + +def test_init_warns_when_pyproject_lacks_engine( + runner: CliRunner, tmp_path: Path +) -> None: + project = tmp_path / "proj" + project.mkdir() + (project / "pyproject.toml").write_text( + '[project]\nname = "p"\nversion = "0"\ndependencies = ["numpy"]\n' + ) + result = runner.invoke(main, ["init", str(project), "--no-git", "--json"]) + assert result.exit_code == 0, result.output + report = json.loads(result.output) + assert any("lightcone-cli" in w for w in report["warnings"]) def test_init_is_idempotent(runner: CliRunner, tmp_path: Path) -> None: """A second run reports everything unchanged and rewrites nothing.""" project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 0, result.output before = {p: p.read_text() for p in project.rglob("*") if p.is_file()} - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--json"] - ) + result = runner.invoke(main, ["init", str(project), "--no-git", "--json"]) assert result.exit_code == 0, result.output report = json.loads(result.output) assert report["converged"] is True @@ -136,8 +252,9 @@ def test_init_is_idempotent(runner: CliRunner, tmp_path: Path) -> None: assert report["repaired"] == [] assert {p: p.read_text() for p in project.rglob("*") if p.is_file()} == before - # Gitignore block must not be duplicated across runs. + # Managed blocks must not be duplicated across runs. assert (project / ".gitignore").read_text().count("# lightcone-cli") == 1 + assert (project / "AGENTS.md").read_text().count("") == 1 def test_init_check_reports_drift_without_writing( @@ -145,12 +262,14 @@ def test_init_check_reports_drift_without_writing( ) -> None: project = tmp_path / "proj" result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--check", "--json"] + main, ["init", str(project), "--no-git", "--check", "--json"] ) assert result.exit_code == 1 report = json.loads(result.output) assert report["converged"] is False assert "astra.yaml" in report["created"] + assert "pyproject.toml" in report["created"] + assert "uv.lock" in report["created"] assert not project.exists() # --check writes nothing, not even the dir @@ -158,31 +277,10 @@ def test_init_check_passes_on_converged_project( runner: CliRunner, tmp_path: Path ) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--check"] - ) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 0, result.output - - -def test_init_warns_on_directory_copy(runner: CliRunner, tmp_path: Path) -> None: - """A user Containerfile with a directory COPY is never rewritten, but - the drift is surfaced through the warnings channel.""" - project = tmp_path / "proj" - project.mkdir() - custom = "FROM python:3.12-slim\nRUN apt-get update\nCOPY src/ /app/src/\n" - (project / "Containerfile").write_text(custom) - (project / "src").mkdir() - (project / "src" / "a.py").write_text("a = 1\n") - - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--json"] - ) + result = runner.invoke(main, ["init", str(project), "--no-git", "--check"]) assert result.exit_code == 0, result.output - report = json.loads(result.output) - assert (project / "Containerfile").read_text() == custom - assert any("COPY/ADD of a directory" in w for w in report["warnings"]) def test_init_survives_malformed_lightcone_yaml( @@ -196,7 +294,7 @@ def test_init_survives_malformed_lightcone_yaml( result = runner.invoke( main, - ["init", str(project), "--no-git", "--no-venv", "--scratch", "$SCRATCH", "--json"], + ["init", str(project), "--no-git", "--scratch", "$SCRATCH", "--json"], ) assert result.exit_code == 0, result.output report = json.loads(result.output) @@ -207,105 +305,126 @@ def test_init_survives_malformed_lightcone_yaml( (project / ".lightcone" / "lightcone.yaml").write_text("local\n") result = runner.invoke( main, - ["init", str(project), "--no-git", "--no-venv", "--scratch", "$SCRATCH", "--json"], + ["init", str(project), "--no-git", "--scratch", "$SCRATCH", "--json"], ) assert result.exit_code == 0, result.output -def test_init_points_spec_at_containerfile(runner: CliRunner, tmp_path: Path) -> None: - """The scaffolded spec must reference the project Containerfile — pins - the rewrite against drift in astra's boilerplate image name.""" +def test_init_repairs_missing_piece(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 0, result.output - assert "container: Containerfile" in (project / "astra.yaml").read_text() + (project / ".python-version").unlink() + result = runner.invoke(main, ["init", str(project), "--no-git", "--json"]) + assert result.exit_code == 0, result.output + report = json.loads(result.output) + assert ".python-version" in report["created"] + assert (project / ".python-version").exists() + # The rest was left alone. + assert "astra.yaml" in report["unchanged"] -def test_engine_errors_render_cleanly( - runner: CliRunner, monkeypatch: pytest.MonkeyPatch -) -> None: - """ContainerBuildError from any command surfaces as a one-line CLI error - (exit 1), not a traceback — the group boundary translates it.""" - from lightcone.cli import commands - from lightcone.engine.container import ContainerBuildError - def _boom(*args: object, **kwargs: object) -> Path: - raise ContainerBuildError("COPY of a directory is not supported") +def test_lightcone_requirement_pins_running_version() -> None: + from importlib.metadata import version - monkeypatch.setattr(commands, "_project_root", _boom) - result = runner.invoke(main, ["status"]) - assert result.exit_code == 1 - assert "not supported" in result.output - assert "Traceback" not in result.output + from lightcone.cli.commands import _lightcone_requirement + req = _lightcone_requirement() + v = version("lightcone-cli") + if "dev" in v: + # Dev builds aren't published — unpinned fallback. + assert req == "lightcone-cli" + else: + assert req == f"lightcone-cli=={v}" -def test_cloudbuild_error_is_a_container_build_error() -> None: - """One boundary handler must cover both local and Cloud Build failures.""" - from lightcone.engine.cloudbuild import CloudBuildError - from lightcone.engine.container import ContainerBuildError - assert issubclass(CloudBuildError, ContainerBuildError) +# ---- lc run (probe verb) --------------------------------------------------- -def test_init_repairs_missing_piece(runner: CliRunner, tmp_path: Path) -> None: +def _probe_project(tmp_path: Path, *, with_env: bool = True) -> Path: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - (project / "Containerfile").unlink() - - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--json"] + project.mkdir() + (project / "astra.yaml").write_text( + "outputs:\n - id: best_fit\n recipe:\n command: echo hi\n" ) - assert result.exit_code == 0, result.output - report = json.loads(result.output) - assert "Containerfile" in report["created"] - assert (project / "Containerfile").exists() - # The rest was left alone. - assert "astra.yaml" in report["unchanged"] + if with_env: + (project / "pyproject.toml").write_text(PYPROJECT_MIN) + (project / "uv.lock").write_text(UV_LOCK_MIN) + (project / ".python-version").write_text(PYTHON_VERSION_MIN) + return project + + +def test_run_rename_guard_fires_on_output_id( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`lc run ` is the old pipeline grammar — it must error + with the materialize hint before exec'ing anything.""" + project = _probe_project(tmp_path) + monkeypatch.chdir(project) + result = runner.invoke(main, ["run", "best_fit"]) + assert result.exit_code != 0 + assert "lc materialize best_fit" in result.output + assert "materialized, not run" in result.output + + +def test_run_requires_uv_project( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project = _probe_project(tmp_path, with_env=False) + monkeypatch.chdir(project) + result = runner.invoke(main, ["run", "python", "-V"]) + assert result.exit_code != 0 + assert "pyproject.toml" in result.output -def test_init_venv_uses_uv_when_available( +def test_run_probes_through_uv_run( runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + """The probe delegates to `uv run --locked --exact` from the project + root — byte-for-byte the recipe environment.""" + project = _probe_project(tmp_path) + monkeypatch.chdir(project) calls: list[list[str]] = [] def _fake_run(cmd: list[str], **kwargs: object) -> MagicMock: calls.append(list(cmd)) return MagicMock(returncode=0) - monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/uv" if name == "uv" else None) monkeypatch.setattr(subprocess, "run", _fake_run) - - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git"]) + result = runner.invoke(main, ["run", "python", "-V"]) assert result.exit_code == 0, result.output - - assert ["uv", "venv", "--python", "3.12", ".venv"] in calls - assert [ - "uv", "pip", "install", "--python", ".venv/bin/python", "-r", "requirements.txt", - ] in calls + assert calls, "probe never exec'd" + argv = calls[0] + assert argv[:5] == ["uv", "run", "--locked", "--exact", "--project"] + assert argv[-3:] == ["--", "python", "-V"] -def test_init_venv_falls_back_to_python_when_uv_missing( +def test_run_refuses_containerized_interim( runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - calls: list[list[str]] = [] + project = _probe_project(tmp_path) + (project / "pyproject.toml").write_text( + PYPROJECT_MIN + "\n[tool.lightcone.image]\n" + ) + monkeypatch.chdir(project) + result = runner.invoke(main, ["run", "python", "-V"]) + assert result.exit_code != 0 + assert "containerized" in result.output - def _fake_run(cmd: list[str], **kwargs: object) -> MagicMock: - calls.append(list(cmd)) - return MagicMock(returncode=0) - monkeypatch.setattr(shutil, "which", lambda _: None) - monkeypatch.setattr(subprocess, "run", _fake_run) +# ---- lc build -------------------------------------------------------------- - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git"]) - assert result.exit_code == 0, result.output - assert ["python", "-m", "venv", ".venv"] in calls - assert [ - ".venv/bin/python", "-m", "pip", "install", "-q", "-r", "requirements.txt", - ] in calls +def test_build_direct_mode_is_explanatory_noop( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project = _probe_project(tmp_path) + monkeypatch.chdir(project) + result = runner.invoke(main, ["build"]) + assert result.exit_code == 0, result.output + assert "direct mode" in result.output + assert "[tool.lightcone.image]" in result.output # ---- lc verify ------------------------------------------------------------ @@ -316,20 +435,32 @@ def test_verify_clean_project_returns_zero( ) -> None: """An empty project (no materialized outputs yet) is a clean state, not a verification failure.""" - project = tmp_path / "proj" - project.mkdir() - (project / "astra.yaml").write_text( - "outputs:\n - id: foo\n recipe:\n command: echo\n" - ) + project = _probe_project(tmp_path) monkeypatch.chdir(project) result = runner.invoke(main, ["verify"]) assert result.exit_code == 0 -# ---- lc run command building ------------------------------------------------ +# ---- lc status header ------------------------------------------------------ -def test_run_cmd_inserts_separator_before_targets() -> None: +def test_status_header_lines( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project = _probe_project(tmp_path) + monkeypatch.chdir(project) + result = runner.invoke(main, ["status"]) + assert result.exit_code == 0, result.output + assert "mode:" in result.output + assert "direct" in result.output + assert "image:" in result.output + assert "sandbox:" in result.output + + +# ---- lc materialize command building --------------------------------------- + + +def test_materialize_cmd_inserts_separator_before_targets() -> None: """Regression test for issue #87. snakemake's --rerun-triggers uses nargs=+ so it greedily consumes the @@ -360,7 +491,7 @@ def test_run_cmd_inserts_separator_before_targets() -> None: assert target_idx > sep_idx, "target path must appear after '--'" -def test_run_cmd_no_separator_when_no_targets() -> None: +def test_materialize_cmd_no_separator_when_no_targets() -> None: """When no targets are supplied snakemake runs 'rule all'; '--' is unnecessary.""" from lightcone.cli.commands import _build_snakemake_cmd @@ -377,7 +508,7 @@ def test_run_cmd_no_separator_when_no_targets() -> None: assert "--" not in cmd -def test_run_cmd_multiple_triggers_all_before_separator() -> None: +def test_materialize_cmd_multiple_triggers_all_before_separator() -> None: """All four trigger tokens must precede the '--' separator.""" from lightcone.cli.commands import _build_snakemake_cmd @@ -398,15 +529,10 @@ def test_run_cmd_multiple_triggers_all_before_separator() -> None: assert cmd.index(trigger) < sep_idx, f"trigger '{trigger}' must come before '--'" -# ---- JupyterHub deployment paths ------------------------------------------ - - -def test_run_cmd_uniform_across_backends() -> None: - """One invocation shape for every backend: --shared-fs-usage drops - software-deployment so spawned jobs run plain `python` from the - worker's own environment (the worker image on a gateway, the - driver's activated env locally / on SLURM) instead of embedding the - driver's sys.executable. No gateway-specific flags exist.""" +def test_materialize_cmd_shape() -> None: + """One invocation shape: --shared-fs-usage drops software-deployment + so spawned jobs run plain `python` from the worker's own environment + instead of embedding the driver's sys.executable.""" from lightcone.cli.commands import _build_snakemake_cmd cmd = _build_snakemake_cmd( @@ -427,49 +553,14 @@ def test_run_cmd_uniform_across_backends() -> None: assert cmd.index("--") < cmd.index("results/u/foo/.lightcone-manifest.json") -def test_init_scaffold_is_environment_agnostic( +def test_materialize_refuses_containerized_interim( runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """The scaffold is identical on and off a hub: the Containerfile - carries no environment-specific content (pod identity is deployment - config, not image content), and the image gets the execution stack - — including dask-gateway — via a dedicated lightcone-cli layer.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - hub_project = tmp_path / "hub" - result = runner.invoke(main, ["init", str(hub_project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - - monkeypatch.delenv("DASK_GATEWAY__ADDRESS") - local_project = tmp_path / "local" - result = runner.invoke(main, ["init", str(local_project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - - containerfile = (hub_project / "Containerfile").read_text() - requirements = (hub_project / "requirements.txt").read_text() - assert containerfile == (local_project / "Containerfile").read_text() - assert requirements == (local_project / "requirements.txt").read_text() - assert "useradd" not in containerfile and "USER" not in containerfile - # The execution stack goes in the image, never the venv's - # requirements — `lc` lives outside the project venv. - assert "lightcone-cli" in containerfile - assert "lightcone-cli" not in requirements - - -def test_lightcone_requirement_pins_running_version() -> None: - from importlib.metadata import version - - from lightcone.cli.commands import _lightcone_requirement - - req = _lightcone_requirement() - v = version("lightcone-cli") - if "dev" in v: - # Dev builds aren't published — unpinned fallback. - assert req == "lightcone-cli" - else: - assert req == f"lightcone-cli=={v}" - - -def test_ensure_images_none_runtime_returns_empty(tmp_path: Path) -> None: - from lightcone.cli.commands import _ensure_images - - assert _ensure_images(tmp_path, runtime="none") == [] + project = _probe_project(tmp_path) + (project / "pyproject.toml").write_text( + PYPROJECT_MIN + "\n[tool.lightcone.image]\n" + ) + monkeypatch.chdir(project) + result = runner.invoke(main, ["materialize"]) + assert result.exit_code != 0 + assert "containerized" in result.output diff --git a/tests/test_cloudbuild.py b/tests/test_cloudbuild.py deleted file mode 100644 index 98461095..00000000 --- a/tests/test_cloudbuild.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Unit tests for the GCP Cloud Build backend. - -All GCP surfaces (metadata server, GCS, Cloud Build API, registry) are -mocked at the module's HTTP seams — ``_metadata_access_token`` and -``_request`` — so the tests exercise the real control flow: freshness -probe, staging, submission, polling, failure-tail reporting. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from lightcone.engine import cloudbuild -from lightcone.engine.cloudbuild import ( - BUCKET_ENV, - SERVICE_ACCOUNT_ENV, - CloudBuildError, - cloudbuild_available, - ensure_image, -) -from lightcone.engine.container import REGISTRY_ENV, registry_image_ref - -REGISTRY = "europe-west1-docker.pkg.dev/lightconehub/lightcone-images" - - -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: - for var in (REGISTRY_ENV, BUCKET_ENV, SERVICE_ACCOUNT_ENV): - monkeypatch.delenv(var, raising=False) - - -@pytest.fixture -def deployment(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(REGISTRY_ENV, REGISTRY) - monkeypatch.setenv(BUCKET_ENV, "lightconehub-lightcone-lc-build") - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - (tmp_path / "requirements.txt").write_text("numpy\n") - return tmp_path - - -# ---- backend selection ---------------------------------------------------- - - -def test_unavailable_without_env() -> None: - assert cloudbuild_available() is False - - -def test_available_with_full_contract(deployment: None) -> None: - assert cloudbuild_available() is True - - -def test_unavailable_with_bucket_only(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(BUCKET_ENV, "some-bucket") - assert cloudbuild_available() is False - - -# ---- registry ref / project parsing --------------------------------------- - - -def test_registry_ref_shape(project: Path) -> None: - ref = registry_image_ref("My Proj", project / "Containerfile", project, - registry=REGISTRY + "/") - repo, _, tag = ref.rpartition(":") - assert repo == f"{REGISTRY}/lc-my-proj" - assert len(tag) == 12 - - -def test_gcp_project_from_registry() -> None: - assert cloudbuild._gcp_project(REGISTRY) == "lightconehub" - - -def test_gcp_project_rejects_non_artifact_registry() -> None: - with pytest.raises(CloudBuildError, match="Artifact Registry"): - cloudbuild._gcp_project("ghcr.io/someorg") - - -# ---- ensure_image control flow -------------------------------------------- - - -def _fresh_probe(monkeypatch: pytest.MonkeyPatch, exists: bool | None) -> None: - monkeypatch.setattr(cloudbuild, "registry_image_exists", lambda ref: exists) - - -def test_ensure_image_cached_is_probe_only( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, True) - monkeypatch.setattr( - cloudbuild, "_request", lambda *a, **k: pytest.fail("no HTTP beyond probe") - ) - phases: list[str] = [] - ref = ensure_image( - project, "Containerfile", project_name="proj", - on_progress=lambda p, _d: phases.append(p), - ) - assert ref.startswith(f"{REGISTRY}/lc-proj:") - assert phases == ["cached"] - - -def test_ensure_image_builds_when_absent( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, False) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - monkeypatch.setenv(SERVICE_ACCOUNT_ENV, "builder@lightconehub.iam.gserviceaccount.com") - - calls: list[tuple[str, str]] = [] - poll_status = iter(["WORKING", "SUCCESS"]) - - def fake_request(method: str, url: str, token: str, *, body=None, content_type=""): - calls.append((method, url)) - if "storage.googleapis.com/upload" in url: - return 200, b"{}" - if url.endswith("/builds") and method == "POST": - payload = json.loads(body.decode()) - assert payload["images"][0].startswith(f"{REGISTRY}/lc-proj:") - assert payload["serviceAccount"] == ( - "projects/lightconehub/serviceAccounts/" - "builder@lightconehub.iam.gserviceaccount.com" - ) - assert payload["source"]["storageSource"]["bucket"] == ( - "lightconehub-lightcone-lc-build" - ) - return 200, json.dumps( - {"metadata": {"build": {"id": "build-123"}}} - ).encode() - if "/builds/build-123" in url: - return 200, json.dumps({"status": next(poll_status)}).encode() - raise AssertionError(f"unexpected request {method} {url}") - - monkeypatch.setattr(cloudbuild, "_request", fake_request) - monkeypatch.setattr(cloudbuild.time, "sleep", lambda _s: None) - - phases: list[str] = [] - ref = ensure_image( - project, "Containerfile", project_name="proj", - on_progress=lambda p, _d: phases.append(p), - ) - assert ref.startswith(f"{REGISTRY}/lc-proj:") - assert phases[0] == "staging" - assert "working" in phases and "success" in phases - # Upload happened before submission. - assert "upload" in calls[0][1] - - -def test_ensure_image_failure_surfaces_log_tail( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, False) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - - def fake_request(method: str, url: str, token: str, *, body=None, content_type=""): - if "storage.googleapis.com/upload" in url: - return 200, b"{}" - if url.endswith("/builds") and method == "POST": - return 200, json.dumps( - {"metadata": {"build": {"id": "build-9"}}} - ).encode() - if "/builds/build-9" in url: - return 200, json.dumps({"status": "FAILURE"}).encode() - if "alt=media" in url: - return 200, b"step1 ok\nERROR: pip failed\n" - raise AssertionError(f"unexpected request {method} {url}") - - monkeypatch.setattr(cloudbuild, "_request", fake_request) - monkeypatch.setattr(cloudbuild.time, "sleep", lambda _s: None) - - with pytest.raises(CloudBuildError, match="ERROR: pip failed"): - ensure_image(project, "Containerfile", project_name="proj") - - -def test_ensure_image_force_skips_probe( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr( - cloudbuild, - "registry_image_exists", - lambda ref: pytest.fail("force must skip the freshness probe"), - ) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - - def fake_request(method: str, url: str, token: str, *, body=None, content_type=""): - if "storage.googleapis.com/upload" in url: - return 200, b"{}" - if url.endswith("/builds") and method == "POST": - return 200, json.dumps( - {"metadata": {"build": {"id": "b"}}} - ).encode() - return 200, json.dumps({"status": "SUCCESS"}).encode() - - monkeypatch.setattr(cloudbuild, "_request", fake_request) - ensure_image(project, "Containerfile", project_name="proj", force=True) - - -def test_ensure_image_requires_credentials( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, None) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: None) - with pytest.raises(CloudBuildError, match="Workload Identity"): - ensure_image(project, "Containerfile", project_name="proj") - - -def test_ensure_image_missing_containerfile( - deployment: None, tmp_path: Path -) -> None: - with pytest.raises(CloudBuildError, match="not found"): - ensure_image(tmp_path, "Containerfile", project_name="proj") - - -def test_ensure_image_off_deployment(project: Path) -> None: - with pytest.raises(CloudBuildError, match="not configured for Cloud Build"): - ensure_image(project, "Containerfile", project_name="proj") - - -# ---- staged tarball -------------------------------------------------------- - - -def test_staged_tarball_matches_hashed_context(project: Path) -> None: - """The tarball must contain exactly the staged (= hashed) file set.""" - import io - import tarfile - - (project / "results").mkdir() - (project / "results" / "big.bin").write_text("x" * 10) - data = cloudbuild._staged_context_tarball(project, project / "Containerfile") - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: - names = set(tar.getnames()) - assert "Containerfile" in names - assert "requirements.txt" in names - assert not any("results" in n for n in names) - - -# ---- registry probe -------------------------------------------------------- - - -def test_registry_image_exists_parses_ref(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - seen: dict[str, str] = {} - - class _Resp: - status = 200 - - def __enter__(self): - return self - - def __exit__(self, *a: object) -> None: - pass - - def fake_urlopen(req, timeout: int = 0): - seen["url"] = req.full_url - seen["method"] = req.get_method() - return _Resp() - - monkeypatch.setattr(cloudbuild.urllib.request, "urlopen", fake_urlopen) - assert cloudbuild.registry_image_exists(f"{REGISTRY}/lc-proj:abc123") is True - assert seen["method"] == "HEAD" - assert seen["url"] == ( - "https://europe-west1-docker.pkg.dev/v2/" - "lightconehub/lightcone-images/lc-proj/manifests/abc123" - ) - - -def test_registry_image_exists_none_without_credentials( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: None) - assert cloudbuild.registry_image_exists(f"{REGISTRY}/lc-proj:abc") is None diff --git a/tests/test_container.py b/tests/test_container.py deleted file mode 100644 index 25d3b9ed..00000000 --- a/tests/test_container.py +++ /dev/null @@ -1,750 +0,0 @@ -"""Tests for the container runtime layer. - -Covers tag computation, build invocation, runtime detection/config, and -the recipe wrap that the Snakefile generator embeds into ``shell()``. -""" -from __future__ import annotations - -import shlex -from collections.abc import Iterator -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest -import yaml - -from lightcone.engine.container import ( - RUNTIMES, - ContainerBuildError, - build_image, - compute_image_tag, - detect_runtime, - find_dependency_files, - get_container_status, - image_exists_locally, - image_exists_podman_hpc, - is_containerfile, - load_runtime, - pull_image, - resolve_image_for_run, - wrap_recipe, -) - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - """Minimal project with a Containerfile.""" - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - return tmp_path - - -@pytest.fixture -def project_with_deps(project: Path) -> Path: - (project / "requirements.txt").write_text("numpy\npandas\n") - (project / "pyproject.toml").write_text("[project]\nname = 'test'\n") - return project - - -# ---- find_dependency_files / compute_image_tag ---------------------------- - - -class TestFindDependencyFiles: - def test_finds_requirements_txt(self, project: Path) -> None: - (project / "requirements.txt").write_text("numpy\n") - found = find_dependency_files(project) - assert [f.name for f in found] == ["requirements.txt"] - - def test_finds_pyproject_toml(self, project: Path) -> None: - (project / "pyproject.toml").write_text("[project]\n") - found = find_dependency_files(project) - assert [f.name for f in found] == ["pyproject.toml"] - - def test_skips_missing_files(self, project: Path) -> None: - assert find_dependency_files(project) == [] - - def test_finds_multiple(self, project_with_deps: Path) -> None: - names = {f.name for f in find_dependency_files(project_with_deps)} - assert {"requirements.txt", "pyproject.toml"} <= names - - -class TestComputeImageTag: - def test_deterministic(self, project: Path) -> None: - cf = project / "Containerfile" - assert compute_image_tag("test", cf, project) == compute_image_tag("test", cf, project) - - def test_tag_format(self, project: Path) -> None: - tag = compute_image_tag("my-project", project / "Containerfile", project) - assert tag.startswith("lc-my-project-") - assert len(tag.removeprefix("lc-my-project-")) == 12 - - def test_changes_with_containerfile(self, project: Path) -> None: - cf = project / "Containerfile" - tag1 = compute_image_tag("test", cf, project) - cf.write_text("FROM ubuntu:22.04\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_changes_with_requirements(self, project: Path) -> None: - cf = project / "Containerfile" - tag1 = compute_image_tag("test", cf, project) - (project / "requirements.txt").write_text("numpy\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_sanitises_project_name(self, project: Path) -> None: - tag = compute_image_tag("My Project", project / "Containerfile", project) - assert tag.startswith("lc-my-project-") - - def test_changes_with_uv_lock(self, project: Path) -> None: - cf = project / "Containerfile" - tag1 = compute_image_tag("test", cf, project) - (project / "uv.lock").write_text("# v1\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_changes_with_copied_file(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY app.py /app/app.py\n") - (project / "app.py").write_text("print(1)\n") - tag1 = compute_image_tag("test", cf, project) - (project / "app.py").write_text("print(2)\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_directory_copy_source_is_rejected(self, project: Path) -> None: - """The image is an environment, not a code snapshot: directory - COPY sources (COPY src/, COPY . .) raise with guidance instead - of silently baking in a copy nothing executes.""" - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY src/ /app/src/\n") - (project / "src").mkdir() - (project / "src" / "a.py").write_text("a = 1\n") - with pytest.raises(ContainerBuildError, match="directory"): - compute_image_tag("test", cf, project) - - def test_copy_dot_is_rejected(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY . /app/\n") - with pytest.raises(ContainerBuildError, match="not supported"): - compute_image_tag("test", cf, project) - - def test_skips_from_stage_copy(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text( - "FROM python:3.12-slim AS builder\n" - "FROM python:3.12-slim\n" - "COPY --from=builder /tmp/x /app/x\n" - ) - # No real source on host, but parsing must not raise or expand. - tag = compute_image_tag("test", cf, project) - assert tag.startswith("lc-test-") - - def test_skips_url_add(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text( - "FROM python:3.12-slim\nADD https://example.com/x.tgz /app/x.tgz\n" - ) - tag = compute_image_tag("test", cf, project) - assert tag.startswith("lc-test-") - - def test_glob_copy_invalidates_on_match_change(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY *.py /app/\n") - (project / "main.py").write_text("x = 1\n") - tag1 = compute_image_tag("test", cf, project) - (project / "main.py").write_text("x = 2\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_swap_dep_file_names_not_collision(self, project: Path) -> None: - # Same total bytes, swapped between two dep files: must not collide - # (the old concat-without-delimiter scheme would have). - (project / "requirements.txt").write_text("numpy\n") - (project / "requirements-dev.txt").write_text("pandas\n") - tag1 = compute_image_tag("test", project / "Containerfile", project) - (project / "requirements.txt").write_text("pandas\n") - (project / "requirements-dev.txt").write_text("numpy\n") - tag2 = compute_image_tag("test", project / "Containerfile", project) - assert tag1 != tag2 - - -# ---- image_exists_locally / image_exists_podman_hpc ----------------------- - - -class TestImageExistsLocally: - @patch("lightcone.engine.container.subprocess.run") - def test_docker_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0) - assert image_exists_locally("lc-foo", runtime="docker") is True - assert mock_run.call_args[0][0][0] == "docker" - - @patch("lightcone.engine.container.subprocess.run") - def test_podman_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0) - assert image_exists_locally("lc-foo", runtime="podman") is True - assert mock_run.call_args[0][0][0] == "podman" - - @patch("lightcone.engine.container.subprocess.run") - def test_not_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=1) - assert image_exists_locally("lc-foo", runtime="docker") is False - - @patch("lightcone.engine.container.subprocess.run", side_effect=FileNotFoundError) - def test_runtime_not_installed(self, mock_run: MagicMock) -> None: - assert image_exists_locally("lc-foo", runtime="docker") is False - - @patch("lightcone.engine.container.image_exists_podman_hpc", return_value=True) - def test_podman_hpc_delegates(self, mock_phpc: MagicMock) -> None: - assert image_exists_locally("lc-foo", runtime="podman-hpc") is True - mock_phpc.assert_called_once_with("lc-foo") - - -class TestImageExistsPodmanHpc: - @patch("lightcone.engine.container.subprocess.run") - def test_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0) - assert image_exists_podman_hpc("img:v1") is True - - @patch("lightcone.engine.container.subprocess.run", side_effect=FileNotFoundError) - def test_not_installed(self, mock_run: MagicMock) -> None: - assert image_exists_podman_hpc("img:v1") is False - - -# ---- build_image ---------------------------------------------------------- - - -class TestBuildImage: - @patch("lightcone.engine.container.subprocess.run") - def test_docker_success(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="ok", stderr="") - result = build_image("lc-test", project / "Containerfile", project, runtime="docker") - assert result.tag == "lc-test" - cmd = mock_run.call_args[0][0] - assert cmd[0] == "docker" - assert cmd[1] == "build" - - @patch("lightcone.engine.container.subprocess.run") - def test_podman_success(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - result = build_image("lc-test", project / "Containerfile", project, runtime="podman") - assert result.tag == "lc-test" - assert mock_run.call_args[0][0][0] == "podman" - - @patch("lightcone.engine.container._podman_hpc_migrate") - @patch("lightcone.engine.container.subprocess.run") - def test_podman_hpc_migrates_after_build( - self, mock_run: MagicMock, mock_migrate: MagicMock, project: Path - ) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - build_image("lc-test", project / "Containerfile", project, runtime="podman-hpc") - mock_migrate.assert_called_once_with("lc-test") - - @patch("lightcone.engine.container.subprocess.run") - def test_failure_raises(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="boom") - with pytest.raises(ContainerBuildError, match="docker build failed"): - build_image("lc-test", project / "Containerfile", project, runtime="docker") - - @patch("lightcone.engine.container.subprocess.run", side_effect=FileNotFoundError) - def test_runtime_missing_raises(self, mock_run: MagicMock, project: Path) -> None: - with pytest.raises(ContainerBuildError, match="podman is not installed"): - build_image("lc-test", project / "Containerfile", project, runtime="podman") - - def test_unsupported_runtime_raises(self, project: Path) -> None: - with pytest.raises(ContainerBuildError, match="Unsupported build runtime"): - build_image( - "lc-test", project / "Containerfile", project, runtime="apptainer" - ) - - @patch("lightcone.engine.container.subprocess.run") - def test_build_args(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - build_image( - "lc-test", - project / "Containerfile", - project, - runtime="docker", - build_args={"PY_VERSION": "3.12"}, - ) - cmd = mock_run.call_args[0][0] - assert "--build-arg" in cmd - assert "PY_VERSION=3.12" in cmd - - def test_build_stages_context_off_source_tree(self, project: Path) -> None: - """Build context must be a tempdir, not the source project. - - On NERSC, projects living on DVS-mounted home/CFS hit - ``llistxattr EPROTO`` when buildah's copier walks COPY sources. - Staging into ``$TMPDIR`` (tmpfs) is what lets builds succeed there. - """ - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY app.py /app/app.py\n") - (project / "app.py").write_text("print('hi')\n") - - captured: dict = {} - - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd - ctx = Path(cmd[-1]) - captured["ctx"] = ctx - captured["files"] = sorted( - p.relative_to(ctx).as_posix() - for p in ctx.rglob("*") - if p.is_file() - ) - return MagicMock(returncode=0, stdout="", stderr="") - - with patch( - "lightcone.engine.container.subprocess.run", side_effect=fake_run - ): - build_image("lc-test", cf, project, runtime="podman") - - assert captured["ctx"].resolve() != project.resolve() - assert not captured["ctx"].exists() - assert "Containerfile" in captured["files"] - assert "app.py" in captured["files"] - - def test_build_rejects_copy_dot(self, project: Path) -> None: - """A ``COPY . .`` Containerfile fails the build with guidance - before any runtime is invoked.""" - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY . /app/\n") - with pytest.raises(ContainerBuildError, match="environment"): - build_image("lc-test", cf, project, runtime="podman") - - @patch("lightcone.engine.container.subprocess.run") - def test_build_cleans_stage_on_failure( - self, mock_run: MagicMock, project: Path - ) -> None: - """Staged tempdir is removed even when the build fails.""" - mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="boom") - with pytest.raises(ContainerBuildError): - build_image("lc-test", project / "Containerfile", project, runtime="docker") - ctx = Path(mock_run.call_args[0][0][-1]) - assert not ctx.exists() - - -# ---- pull_image ----------------------------------------------------------- - - -class TestPullImage: - @patch("lightcone.engine.container.subprocess.run") - def test_pull_success_docker(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - pull_image("python:3.12-slim", runtime="docker") - cmd = mock_run.call_args[0][0] - assert cmd == ["docker", "pull", "python:3.12-slim"] - - @patch("lightcone.engine.container.subprocess.run") - def test_pull_success_podman(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - pull_image("python:3.12-slim", runtime="podman") - assert mock_run.call_args[0][0][0] == "podman" - - @patch("lightcone.engine.container._podman_hpc_migrate") - @patch("lightcone.engine.container.subprocess.run") - def test_pull_podman_hpc_migrates( - self, mock_run: MagicMock, mock_migrate: MagicMock - ) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - pull_image("python:3.12-slim", runtime="podman-hpc") - mock_migrate.assert_called_once_with("python:3.12-slim") - - @patch("lightcone.engine.container.subprocess.run") - def test_pull_failure_raises(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="boom") - with pytest.raises(ContainerBuildError, match="docker pull"): - pull_image("python:3.12-slim", runtime="docker") - - def test_unsupported_runtime_raises(self) -> None: - with pytest.raises(ContainerBuildError, match="Unsupported runtime"): - pull_image("img", runtime="apptainer") - - -# ---- detect_runtime / load_runtime --------------------------------------- - - -class TestDetectRuntime: - @pytest.fixture(autouse=True) - def _generic_hostname(self) -> Iterator[None]: - # Pin hostname to one that doesn't match any site so the default - # RUNTIMES order applies. Site-aware behaviour is exercised in - # TestSiteAwareDetection below. - with patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="generic-laptop", - ): - yield - - @patch("lightcone.engine.container.shutil.which") - def test_podman_hpc_preferred_when_present(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: f"/usr/bin/{name}" - assert detect_runtime() == "podman-hpc" - - @patch("lightcone.engine.container.shutil.which") - def test_podman_preferred_over_docker(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: ( - None if name == "podman-hpc" else f"/usr/bin/{name}" - ) - assert detect_runtime() == "podman" - - @patch("lightcone.engine.container.shutil.which") - def test_docker_only(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: "/usr/bin/docker" if name == "docker" else None - with patch( - "lightcone.engine.container._docker_daemon_up", return_value=True - ): - assert detect_runtime() == "docker" - - @patch("lightcone.engine.container.shutil.which") - def test_docker_skipped_when_daemon_down(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: "/usr/bin/docker" if name == "docker" else None - with patch( - "lightcone.engine.container._docker_daemon_up", return_value=False - ): - assert detect_runtime() is None - - @patch("lightcone.engine.container.shutil.which") - def test_docker_daemon_down_falls_through_to_podman( - self, mock_which: MagicMock - ) -> None: - mock_which.side_effect = lambda name: ( - None if name == "podman-hpc" else f"/usr/bin/{name}" - ) - with patch( - "lightcone.engine.container._docker_daemon_up", return_value=False - ): - assert detect_runtime() == "podman" - - @patch("lightcone.engine.container.shutil.which", return_value=None) - def test_none_available(self, mock_which: MagicMock) -> None: - assert detect_runtime() is None - - def test_no_apptainer(self) -> None: - # Apptainer/singularity must NOT be in the supported runtimes list — - # we own container invocation and only support OCI runtimes. - assert "apptainer" not in RUNTIMES - assert "singularity" not in RUNTIMES - - -class TestSiteAwareDetection: - @patch("lightcone.engine.container.shutil.which") - @patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="login29.chn.perlmutter.nersc.gov", - ) - def test_perlmutter_picks_podman_hpc( - self, _hostname: MagicMock, mock_which: MagicMock - ) -> None: - mock_which.side_effect = lambda name: f"/usr/bin/{name}" - assert detect_runtime() == "podman-hpc" - - @patch("lightcone.engine.container.shutil.which") - @patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="login29.chn.perlmutter.nersc.gov", - ) - def test_falls_through_when_site_runtime_missing( - self, _hostname: MagicMock, mock_which: MagicMock - ) -> None: - # Site preference is a hint — explicit user config goes through - # load_runtime, which DOES error on missing binary. - mock_which.side_effect = lambda name: ( - None if name == "podman-hpc" else f"/usr/bin/{name}" - ) - assert detect_runtime() == "podman" - - @patch("lightcone.engine.container.shutil.which") - @patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="generic-laptop", - ) - def test_unknown_site_uses_default_order( - self, _hostname: MagicMock, mock_which: MagicMock - ) -> None: - mock_which.side_effect = lambda name: f"/usr/bin/{name}" - assert detect_runtime() == RUNTIMES[0] - - -class TestLoadRuntime: - def _write_config(self, tmp_path: Path, content: dict) -> None: - cfg_dir = tmp_path / ".lightcone" - cfg_dir.mkdir() - (cfg_dir / "config.yaml").write_text(yaml.safe_dump(content)) - - def test_no_config_uses_auto( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.detect_runtime", lambda: "docker" - ) - choice = load_runtime() - assert choice.runtime == "docker" - assert choice.explicit is False - - def test_auto_with_no_runtime_returns_none_implicitly( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """auto + nothing on PATH → none, but explicit=False so the - caller can warn that this is a silent fallback.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.detect_runtime", lambda: None - ) - choice = load_runtime() - assert choice.runtime == "none" - assert choice.explicit is False - - def test_explicit_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """User opted out of containers — explicit=True, no warnings owed.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - self._write_config(tmp_path, {"container": {"runtime": "none"}}) - choice = load_runtime() - assert choice.runtime == "none" - assert choice.explicit is True - - def test_explicit_runtime_present( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.shutil.which", - lambda name: f"/usr/bin/{name}" if name == "podman" else None, - ) - self._write_config(tmp_path, {"container": {"runtime": "podman"}}) - choice = load_runtime() - assert choice.runtime == "podman" - assert choice.explicit is True - - def test_explicit_runtime_missing_on_path_raises( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.shutil.which", lambda _: None - ) - self._write_config(tmp_path, {"container": {"runtime": "podman"}}) - with pytest.raises(ContainerBuildError, match="not on PATH"): - load_runtime() - - def test_unknown_runtime_raises( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - self._write_config(tmp_path, {"container": {"runtime": "apptainer"}}) - with pytest.raises(ContainerBuildError, match="Unknown container.runtime"): - load_runtime() - - -# ---- resolve_image_for_run ----------------------------------------------- - - -class TestResolveImageForRun: - def test_none_returns_none(self, project: Path) -> None: - assert resolve_image_for_run( - None, project_path=project, project_name="test" - ) is None - - def test_registry_image_passes_through(self, project: Path) -> None: - assert resolve_image_for_run( - "python:3.12-slim", project_path=project, project_name="test" - ) == "python:3.12-slim" - - def test_namespaced_registry_image_passes_through(self, project: Path) -> None: - assert resolve_image_for_run( - "ghcr.io/foo/bar:tag", project_path=project, project_name="test" - ) == "ghcr.io/foo/bar:tag" - - def test_containerfile_resolves_to_tag(self, project: Path) -> None: - result = resolve_image_for_run( - "Containerfile", project_path=project, project_name="test" - ) - assert result is not None - assert result.startswith("lc-test-") - - -# ---- wrap_recipe ---------------------------------------------------------- - - -class TestWrapRecipe: - def test_no_image_passthrough(self) -> None: - assert wrap_recipe("echo hi", image=None, runtime="podman") == "echo hi" - - def test_runtime_none_passthrough(self) -> None: - assert wrap_recipe( - "echo hi", image="python:3.12-slim", runtime="none" - ) == "echo hi" - - def test_podman_wrap_basic(self) -> None: - wrapped = wrap_recipe( - "echo hi", image="python:3.12-slim", runtime="podman" - ) - assert wrapped.startswith("podman run --rm --pull=never ") - assert "python:3.12-slim" in wrapped - # The recipe is shell-quoted to survive nested shells. - assert shlex.quote("echo hi") in wrapped - - def test_docker_wrap(self) -> None: - wrapped = wrap_recipe("echo hi", image="img:v1", runtime="docker") - assert wrapped.startswith("docker run --rm --pull=never ") - - def test_podman_hpc_wrap(self) -> None: - wrapped = wrap_recipe("echo hi", image="img:v1", runtime="podman-hpc") - assert wrapped.startswith("podman-hpc run --rm --pull=never ") - - def test_pull_never_short_name_safe(self) -> None: - """``--pull=never`` is what makes locally-built short-name images - like ``lc-foo-abc123`` work under podman, which would otherwise - try to resolve the name against unqualified-search-registries.""" - wrapped = wrap_recipe( - "echo", image="lc-foo-abc123", runtime="podman" - ) - assert "--pull=never" in wrapped - - def test_preserves_snakemake_placeholders(self) -> None: - """Snakemake's ``{output[0]}`` must survive the wrap so it can - substitute at exec time.""" - wrapped = wrap_recipe( - "echo > {output[0]}/x", image="img:v1", runtime="podman" - ) - assert "{output[0]}" in wrapped - - def test_preserves_recipe_with_single_quotes(self) -> None: - """Recipes may contain single quotes (e.g. ``python -c 'print(1)'``). - The shlex.quote escape must survive nested shell parsing.""" - recipe = """python -c 'print("hi")'""" - wrapped = wrap_recipe(recipe, image="img:v1", runtime="podman") - # Round-trip through shlex.split should yield the original recipe - # as the last argument (the bash -c argument). - tokens = shlex.split(wrapped) - assert tokens[-1] == recipe - - def test_unsupported_runtime_raises(self) -> None: - with pytest.raises(ContainerBuildError, match="Unsupported run runtime"): - wrap_recipe("echo", image="img:v1", runtime="apptainer") - - def test_bind_mounts_pwd(self) -> None: - """Recipes that write to relative paths need $PWD bind-mounted.""" - wrapped = wrap_recipe("echo", image="img:v1", runtime="podman") - assert '-v "$PWD":"$PWD"' in wrapped - assert '-w "$PWD"' in wrapped - - -# ---- get_container_status ------------------------------------------------- - - -class TestGetContainerStatus: - def test_none(self, project: Path) -> None: - s = get_container_status(None, project, "test", runtime="docker") - assert s.type == "none" - - def test_prebuilt(self, project: Path) -> None: - s = get_container_status("python:3.12", project, "test", runtime="docker") - assert s.type == "prebuilt" - assert s.image == "python:3.12" - - @patch("lightcone.engine.container.image_exists_locally", return_value=False) - def test_containerfile_not_built( - self, mock_exists: MagicMock, project: Path - ) -> None: - s = get_container_status("Containerfile", project, "test", runtime="docker") - assert s.type == "build" - assert s.exists is False - assert s.image is not None - - @patch("lightcone.engine.container.image_exists_locally", return_value=True) - def test_containerfile_built( - self, mock_exists: MagicMock, project: Path - ) -> None: - s = get_container_status("Containerfile", project, "test", runtime="docker") - assert s.type == "build" - assert s.exists is True - - def test_runtime_none_skips_existence_check(self, project: Path) -> None: - s = get_container_status("Containerfile", project, "test", runtime="none") - assert s.type == "build" - assert s.exists is None - - -# ---- is_containerfile ----------------------------------------------------- - - -class TestIsContainerfile: - def test_existing_file(self, project: Path) -> None: - assert is_containerfile("Containerfile", project) is True - - def test_missing_file(self, project: Path) -> None: - assert is_containerfile("python:3.12-slim", project) is False - - -# ---- kubernetes runtime --------------------------------------------------- - - -class TestKubernetesRuntime: - def test_wrap_recipe_is_passthrough(self) -> None: - """The worker pod already runs the image — wrapping would - containerize twice.""" - from lightcone.engine.container import KUBERNETES - - wrapped = wrap_recipe( - "python run.py", image="reg/lc-p:abc", runtime=KUBERNETES - ) - assert wrapped == "python run.py" - - def test_registry_ref_shares_identity_with_local_tag( - self, project: Path - ) -> None: - from lightcone.engine.container import registry_image_ref - - tag = compute_image_tag("proj", project / "Containerfile", project) - ref = registry_image_ref( - "proj", project / "Containerfile", project, registry="reg.io/ns/repo" - ) - digest = tag.rsplit("-", 1)[1] - assert ref == f"reg.io/ns/repo/lc-proj:{digest}" - - def test_resolve_image_uses_registry_when_given(self, project: Path) -> None: - ref = resolve_image_for_run( - "Containerfile", - project_path=project, - project_name="proj", - registry="reg.io/ns/repo", - ) - assert ref is not None and ref.startswith("reg.io/ns/repo/lc-proj:") - - def test_resolve_prebuilt_ignores_registry(self, project: Path) -> None: - assert ( - resolve_image_for_run( - "python:3.12-slim", - project_path=project, - project_name="proj", - registry="reg.io/ns/repo", - ) - == "python:3.12-slim" - ) - - def test_detect_runtime_site_kubernetes_skips_path_probe( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """A gateway deployment has no OCI binary to find — the site - preference short-circuits detection entirely.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dg") - monkeypatch.setattr( - "lightcone.engine.container.shutil.which", - lambda _: pytest.fail("no PATH probing on kubernetes sites"), - ) - assert detect_runtime() == "kubernetes" - - def test_load_runtime_explicit_kubernetes( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - cfg_dir = tmp_path / ".lightcone" - cfg_dir.mkdir() - (cfg_dir / "config.yaml").write_text( - yaml.safe_dump({"container": {"runtime": "kubernetes"}}) - ) - choice = load_runtime() - assert choice.runtime == "kubernetes" - assert choice.explicit is True diff --git a/tests/test_dask_cluster.py b/tests/test_dask_cluster.py index 099ee77b..987ab380 100644 --- a/tests/test_dask_cluster.py +++ b/tests/test_dask_cluster.py @@ -1,16 +1,6 @@ -"""Unit tests for the cluster bootstrap. - -We test the routing decision (which branch fires given env vars) and the -node-shape detection. The actual `LocalCluster` spin-up is exercised in a -single smoke test; the `srun`-backed path is mocked because real -multi-node testing requires SLURM. -""" - +"""Tests for the run-scoped LocalCluster lifecycle.""" from __future__ import annotations -from contextlib import contextmanager -from unittest.mock import patch - import pytest from lightcone.engine.dask_cluster import ( @@ -19,586 +9,43 @@ RESOURCE_MEMORY, _detect_node_shape, _NodeShape, - _resources_arg, + _resource_dict, cluster_for_run, ) -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: - for var in ( - "DASK_SCHEDULER_ADDRESS", - "DASK_GATEWAY__ADDRESS", - "LIGHTCONE_GATEWAY_WORKER_TIMEOUT", - "SLURM_JOB_ID", - "SLURM_NNODES", - "SLURM_CPUS_ON_NODE", - "SLURM_MEM_PER_NODE", - "SLURM_GPUS_ON_NODE", - ): - monkeypatch.delenv(var, raising=False) - - -def test_detect_shape_falls_back_to_os(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("os.cpu_count", lambda: 8) - shape = _detect_node_shape() - assert shape.cpus == 8 - assert shape.gpus == 0 - - -def test_detect_shape_reads_slurm_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SLURM_CPUS_ON_NODE", "64") - monkeypatch.setenv("SLURM_MEM_PER_NODE", "256000") # 256 GB in MB - monkeypatch.setenv("SLURM_GPUS_ON_NODE", "4") - shape = _detect_node_shape() - assert shape.cpus == 64 - assert shape.mem_bytes == 256_000_000_000 - assert shape.gpus == 4 - - -def test_resources_arg_minimal() -> None: - arg = _resources_arg(_NodeShape(cpus=8, mem_bytes=0, gpus=0)) - assert arg == "cpus=8" - - -def test_resources_arg_full() -> None: - arg = _resources_arg(_NodeShape(cpus=64, mem_bytes=256_000_000_000, gpus=4)) - assert arg == "cpus=64 memory=256000000000 gpus=4" - - -def test_existing_scheduler_address_yields_unchanged( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://example:8786") - - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://example:8786"} - - -def test_no_env_uses_local_cluster() -> None: - """The local-cluster branch should actually start a (tiny) cluster.""" - sentinel: dict[str, str] = {} - - @contextmanager - def _fake_local(*, verbose: bool, local_directory: str | None = None): - sentinel["called"] = "local" - yield "tcp://stub:9999" - - with patch("lightcone.engine.dask_cluster._local_cluster", _fake_local): - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://stub:9999"} - assert sentinel["called"] == "local" - - -def test_slurm_env_takes_slurm_path(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SLURM_JOB_ID", "12345") - sentinel: dict[str, str] = {} - - @contextmanager - def _fake_slurm(*, verbose: bool, local_directory: str | None = None): - sentinel["called"] = "slurm" - yield "tcp://stub:9999" - - with patch("lightcone.engine.dask_cluster._slurm_backed_cluster", _fake_slurm): - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://stub:9999"} - assert sentinel["called"] == "slurm" - - -def test_existing_scheduler_address_wins_over_slurm( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """If both are set, the explicit address takes precedence.""" - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://existing:8786") - monkeypatch.setenv("SLURM_JOB_ID", "12345") - - @contextmanager - def _should_not_run(*, verbose: bool, local_directory: str | None = None): - raise AssertionError("slurm path should not have been taken") - yield # pragma: no cover - - with patch("lightcone.engine.dask_cluster._slurm_backed_cluster", _should_not_run): - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://existing:8786"} - - -def test_slurm_backed_cluster_binds_to_routable_host( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Multi-node SLURM allocations need the scheduler bound to a hostname - workers on other nodes can reach. The default LocalCluster host of - 127.0.0.1 fails silently with `wait_for_workers` timeouts. - """ - monkeypatch.setenv("SLURM_JOB_ID", "12345") - monkeypatch.setenv("SLURM_NNODES", "2") - monkeypatch.setenv("SLURMD_NODENAME", "nid001234") - monkeypatch.setattr( - "lightcone.engine.dask_cluster.shutil.which", lambda _: "/usr/bin/dask" - ) - - captured: dict[str, object] = {} - - class _FakeCluster: - def __init__(self, **kwargs: object) -> None: - captured.update(kwargs) - self.scheduler_address = "tcp://nid001234:8786" - - def close(self) -> None: - pass - - class _FakeClient: - def __init__(self, addr: str) -> None: - captured["client_addr"] = addr - - def wait_for_workers(self, n_workers: int, timeout: int) -> None: - pass - - def close(self) -> None: - pass +class TestNodeShape: + def test_detects_cpus(self) -> None: + shape = _detect_node_shape() + assert shape.cpus >= 1 - class _FakePopen: - def __init__(self, cmd: list[str], **kwargs: object) -> None: - captured["worker_cmd"] = cmd - captured["worker_kwargs"] = kwargs + def test_resource_dict_always_advertises_cpus(self) -> None: + res = _resource_dict(_NodeShape(cpus=4, mem_bytes=0, gpus=0)) + assert res == {RESOURCE_CPUS: 4.0} - def terminate(self) -> None: - pass + def test_resource_dict_includes_memory_when_known(self) -> None: + res = _resource_dict(_NodeShape(cpus=2, mem_bytes=8_000_000_000, gpus=0)) + assert res[RESOURCE_MEMORY] == 8_000_000_000.0 - def wait(self, timeout: int | None = None) -> int: - return 0 - - def kill(self) -> None: - pass - - monkeypatch.setattr("dask.distributed.LocalCluster", _FakeCluster) - monkeypatch.setattr("dask.distributed.Client", _FakeClient) - monkeypatch.setattr("subprocess.Popen", _FakePopen) - - from lightcone.engine.dask_cluster import _slurm_backed_cluster - - with _slurm_backed_cluster(verbose=False, local_directory=None) as addr: - assert addr == "tcp://nid001234:8786" - - assert captured.get("host") == "nid001234", ( - f"LocalCluster must be told to bind to the SLURM nodename so remote " - f"workers can connect; got host={captured.get('host')!r}" - ) - - -def test_slurm_backed_cluster_falls_back_to_gethostname( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Without SLURMD_NODENAME, fall back to socket.gethostname().""" - monkeypatch.setenv("SLURM_JOB_ID", "12345") - monkeypatch.setenv("SLURM_NNODES", "1") - monkeypatch.delenv("SLURMD_NODENAME", raising=False) - monkeypatch.setattr( - "lightcone.engine.dask_cluster.shutil.which", lambda _: "/usr/bin/dask" - ) - monkeypatch.setattr( - "lightcone.engine.dask_cluster.socket.gethostname", lambda: "host-fallback" - ) - - captured: dict[str, object] = {} - - class _FakeCluster: - def __init__(self, **kwargs: object) -> None: - captured.update(kwargs) - self.scheduler_address = "tcp://host-fallback:8786" - - def close(self) -> None: - pass - - class _FakeClient: - def __init__(self, addr: str) -> None: - pass - - def wait_for_workers(self, n_workers: int, timeout: int) -> None: - pass - - def close(self) -> None: - pass - - class _FakePopen: - def __init__(self, cmd: list[str], **kwargs: object) -> None: - pass - - def terminate(self) -> None: - pass - - def wait(self, timeout: int | None = None) -> int: - return 0 - - def kill(self) -> None: - pass - - monkeypatch.setattr("dask.distributed.LocalCluster", _FakeCluster) - monkeypatch.setattr("dask.distributed.Client", _FakeClient) - monkeypatch.setattr("subprocess.Popen", _FakePopen) - - from lightcone.engine.dask_cluster import _slurm_backed_cluster - - with _slurm_backed_cluster(verbose=False, local_directory=None): - pass - - assert captured.get("host") == "host-fallback" - - -def test_local_cluster_advertises_memory_and_gpus( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Dask only schedules a task on a worker that advertises every - requested resource key — so the local worker must expose mem and - gpus too, otherwise rules with ``mem_mb``/``gpus_per_task`` hang. - """ - monkeypatch.setattr( - "lightcone.engine.dask_cluster._detect_node_shape", - lambda: _NodeShape(cpus=4, mem_bytes=16_000_000_000, gpus=2), - ) - - captured: dict[str, object] = {} - - class _FakeCluster: - def __init__(self, **kwargs: object) -> None: - captured.update(kwargs) - self.scheduler_address = "tcp://stub:0" - - def close(self) -> None: - pass - - monkeypatch.setattr("dask.distributed.LocalCluster", _FakeCluster) - - from lightcone.engine.dask_cluster import _local_cluster - - with _local_cluster(verbose=False, local_directory=None): - pass - - resources = captured.get("resources") - assert isinstance(resources, dict) - assert set(resources.keys()) == {RESOURCE_CPUS, RESOURCE_MEMORY, RESOURCE_GPUS} + def test_resource_dict_includes_gpus_when_present(self) -> None: + res = _resource_dict(_NodeShape(cpus=2, mem_bytes=0, gpus=1)) + assert res[RESOURCE_GPUS] == 1.0 @pytest.mark.slow -def test_local_cluster_smoke() -> None: - """End-to-end: a real LocalCluster spins up, accepts a task, tears down.""" - from dask.distributed import Client - - from lightcone.engine.dask_cluster import _local_cluster - - with _local_cluster(verbose=False, local_directory=None) as addr: - client = Client(addr) - try: - assert client.submit(lambda x: x + 1, 41).result() == 42 - finally: - client.close() - -# --------------------------------------------------------------------------- -# Dask Gateway branch -# --------------------------------------------------------------------------- - - -def _install_fake_gateway( - monkeypatch: pytest.MonkeyPatch, - record: dict[str, object], - *, - worker_resources: dict[str, float] | None = None, - wait_raises: bool = False, - declared_options: dict[str, object] | None = None, -): - """Register a fake ``dask_gateway`` module and return it.""" - import sys - from types import SimpleNamespace - - resources = ( - worker_resources - if worker_resources is not None - else {"cpus": 2.0, "memory": 4e9} - ) - declared = ( - declared_options - if declared_options is not None - else { - "image": "notebook:latest", - "worker_cores": 2, - "worker_memory": 4.0, - "environment": {}, - } - ) - - class _FakeClient: - def wait_for_workers(self, n_workers: int, timeout: int) -> None: - record["waited"] = (n_workers, timeout) - if wait_raises: - raise TimeoutError("no workers") - - def scheduler_info(self) -> dict[str, object]: - return {"workers": {"w0": {"resources": resources}}} - - def close(self) -> None: - record["client_closed"] = True - - class _FakeCluster: - name = "hub.abc123" - dashboard_link = "http://dash" - - def adapt(self, minimum: int, maximum: int) -> None: - record["adapt"] = (minimum, maximum) - - def get_client(self) -> _FakeClient: - return _FakeClient() - - def shutdown(self) -> None: - record["shutdown"] = True - - def close(self) -> None: - record["closed"] = True - - class _FakeGateway: - def cluster_options(self) -> dict[str, object]: - return dict(declared) - - def new_cluster(self, shutdown_on_close: bool = True, **options: object): - record["shutdown_on_close"] = shutdown_on_close - record["options"] = options - return _FakeCluster() - - module = SimpleNamespace(Gateway=_FakeGateway) - monkeypatch.setitem(sys.modules, "dask_gateway", module) - return module - - -def test_gateway_env_takes_gateway_branch(monkeypatch: pytest.MonkeyPatch) -> None: - from lightcone.engine.dask_cluster import GATEWAY_CLUSTER_ENV - - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with cluster_for_run(worker_image="reg/lc-p:abc", max_workers=4) as env: - assert env == {GATEWAY_CLUSTER_ENV: "hub.abc123"} - opts = record["options"] - assert opts["image"] == "reg/lc-p:abc" # type: ignore[index] - assert record["adapt"] == (1, 4) - assert "shutdown" not in record - - assert record["shutdown"] is True, "run-scoped cluster must be culled on exit" - - -def test_gateway_without_image_uses_deployment_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with cluster_for_run() as _env: - pass - - assert "image" not in record["options"], "no image option → deployment default" # type: ignore[operator] - - -def test_explicit_scheduler_address_wins_over_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://existing:8786") - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://existing:8786"} - - -def test_gateway_culled_when_body_raises(monkeypatch: pytest.MonkeyPatch) -> None: - """The cluster must be shut down even when the run fails.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with pytest.raises(RuntimeError, match="boom"): - with cluster_for_run(): - raise RuntimeError("boom") - - assert record["shutdown"] is True - - -def test_gateway_zero_workers_fails_loudly_and_culls( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setenv("LIGHTCONE_GATEWAY_WORKER_TIMEOUT", "7") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record, wait_raises=True) - - with pytest.raises(RuntimeError, match="within 7s"): - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass # pragma: no cover - - assert record["waited"] == (1, 7) - assert record["shutdown"] is True - - -def test_gateway_missing_resource_contract_refused( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record, worker_resources={}) - - with pytest.raises(RuntimeError, match="resource contract"): - with cluster_for_run(): - pass # pragma: no cover - - assert record["shutdown"] is True - - -def test_gateway_branch_active_matches_routing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from lightcone.engine.dask_cluster import gateway_branch_active - - assert gateway_branch_active() is False - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - assert gateway_branch_active() is True - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://existing:8786") - assert gateway_branch_active() is False - - -def test_gateway_explicit_image_beats_ambient_default() -> None: - """The deployment injects DASK_GATEWAY__CLUSTER__OPTIONS__IMAGE - (= the notebook image) as the client's ambient default. lc run's - explicit ``image`` kwarg MUST override it — otherwise every cluster - would run the notebook image instead of the one `lc build` just - produced. Pinned against the real dask-gateway client merge logic. - """ - pytest.importorskip("dask_gateway") - import dask - from dask_gateway import Gateway - - captured: dict[str, object] = {} - - async def fake_request(self, method, url, json=None, **kwargs): # type: ignore[no-untyped-def] - captured["cluster_options"] = (json or {}).get("cluster_options") - - class _Resp: - async def json(self) -> dict[str, str]: - return {"name": "hub.fake"} - - return _Resp() - - with dask.config.set({"gateway.cluster.options": {"image": "notebook:latest"}}): - gateway = Gateway(address="http://gateway.invalid", auth="basic") - try: - with patch.object(Gateway, "_request", fake_request): - gateway.submit(image="reg/lc-proj:abc123") - assert captured["cluster_options"] == {"image": "reg/lc-proj:abc123"} - - gateway.submit() - assert captured["cluster_options"] == {"image": "notebook:latest"} - finally: - gateway.close() - - -def test_gateway_self_provisions_worker_environment( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """lc passes everything worker pods need through the STANDARD - `environment` cluster option — resource contract mirrored from the - deployment's declared worker shape, driver identity forwarded, - image ground truth — so the deployment's options handler needs no - lightcone-specific injection.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setenv("HOME", "/home/jovyan") - monkeypatch.setenv("USER", "jovyan") - monkeypatch.setenv("LOGNAME", "jovyan") - record: dict[str, object] = {} - _install_fake_gateway( - monkeypatch, - record, - declared_options={ - "image": "notebook:latest", - "worker_cores": 2, - "worker_memory": 4.0, - "environment": {"EXTRA": "kept"}, - }, - ) - - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass - - env = record["options"]["environment"] # type: ignore[index] - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__CPUS"] == "2" - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] == str(int(4.0 * 1e9)) - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__GPUS"] == "0" - assert env["HOME"] == "/home/jovyan" - assert env["USER"] == "jovyan" - assert env["LOGNAME"] == "jovyan" - assert env["LIGHTCONE_WORKER_IMAGE"] == "reg/lc-p:abc" - assert env["EXTRA"] == "kept", "ambient environment defaults must survive" - - -def test_gateway_provisions_identity_without_user_env( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """USER/LOGNAME must be *derived*, not merely forwarded: notebook - pods often don't export them (their own passwd entry covers - getpass), while the environment-agnostic worker image has NO passwd - entry for the pod uid — so the child snakemake crashes at - ``getpass.getuser()`` unless lc always provisions the vars. - Regression: live run failed with ``getpwuid(): uid not found``.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.delenv("USER", raising=False) - monkeypatch.delenv("LOGNAME", raising=False) - record: dict[str, object] = {} - _install_fake_gateway( - monkeypatch, - record, - declared_options={"image": "notebook:latest", "environment": {}}, - ) - - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass - - import getpass - - env = record["options"]["environment"] # type: ignore[index] - assert env["USER"] == getpass.getuser() - assert env["LOGNAME"] == env["USER"] - - -def test_gateway_worker_image_env_falls_back_to_declared_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with cluster_for_run(): # no project image → deployment default - pass - - env = record["options"]["environment"] # type: ignore[index] - assert env["LIGHTCONE_WORKER_IMAGE"] == "notebook:latest" - - -def test_gateway_no_environment_option_no_injection( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A deployment that doesn't expose `environment` gets no surprise - kwarg (the server would reject it).""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway( - monkeypatch, record, declared_options={"image": "notebook:latest"} - ) - - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass - - assert "environment" not in record["options"] # type: ignore[operator] - - -def test_worker_environment_memory_bytes_heuristic() -> None: - from lightcone.engine.dask_cluster import _worker_environment - - env = _worker_environment({"worker_memory": 4294967296}, None) - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] == "4294967296" - env = _worker_environment({"worker_memory": 4.0}, None) - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] == str(int(4e9)) +class TestLocalCluster: + def test_yields_scheduler_address(self, tmp_path) -> None: # type: ignore[no-untyped-def] + with cluster_for_run(local_directory=str(tmp_path)) as env: + addr = env["DASK_SCHEDULER_ADDRESS"] + assert addr.startswith("tcp://") + + from dask.distributed import Client + + client = Client(addr) + try: + workers = client.scheduler_info()["workers"] + assert len(workers) == 1 + resources = next(iter(workers.values()))["resources"] + assert RESOURCE_CPUS in resources + finally: + client.close() diff --git a/tests/test_dask_plugin.py b/tests/test_dask_plugin.py index 8f77f8c8..803f0528 100644 --- a/tests/test_dask_plugin.py +++ b/tests/test_dask_plugin.py @@ -137,13 +137,6 @@ def test_run_shell_success_drops_noise() -> None: assert block == "" -def test_unpack_result_accepts_legacy_int() -> None: - """Workers running an older lightcone-cli release return a bare int.""" - from snakemake_executor_plugin_dask.executor import _unpack_result - - assert _unpack_result(1) == (1, "") - assert _unpack_result((0, "block\n")) == (0, "block\n") - def test_connect_client_requires_rendezvous( monkeypatch: object, @@ -154,55 +147,10 @@ def test_connect_client_requires_rendezvous( from snakemake_executor_plugin_dask.executor import _connect_client monkeypatch.delenv("DASK_SCHEDULER_ADDRESS", raising=False) # type: ignore[attr-defined] - monkeypatch.delenv("LIGHTCONE_GATEWAY_CLUSTER", raising=False) # type: ignore[attr-defined] - with pytest.raises(WorkflowError, match="LIGHTCONE_GATEWAY_CLUSTER"): + with pytest.raises(WorkflowError, match="DASK_SCHEDULER_ADDRESS"): _connect_client() -def test_connect_client_gateway_rendezvous_by_name( - monkeypatch: object, -) -> None: - """With LIGHTCONE_GATEWAY_CLUSTER set, the executor rejoins the run's - cluster through the Gateway API — never dials gateway:// directly.""" - import sys - from types import SimpleNamespace - - from snakemake_executor_plugin_dask.executor import _connect_client - - record: dict[str, object] = {} - - class _FakeClient: - def close(self) -> None: - record["client_closed"] = True - - class _FakeCluster: - def get_client(self) -> _FakeClient: - return _FakeClient() - - def close(self) -> None: - record["cluster_closed"] = True - - class _FakeGateway: - def connect(self, name: str, shutdown_on_close: bool = True): - record["connected"] = name - record["shutdown_on_close"] = shutdown_on_close - return _FakeCluster() - - monkeypatch.setitem( # type: ignore[attr-defined] - sys.modules, "dask_gateway", SimpleNamespace(Gateway=_FakeGateway) - ) - monkeypatch.setenv("LIGHTCONE_GATEWAY_CLUSTER", "hub.abc") # type: ignore[attr-defined] - monkeypatch.delenv("DASK_SCHEDULER_ADDRESS", raising=False) # type: ignore[attr-defined] - - client, closer = _connect_client() - assert record["connected"] == "hub.abc" - assert record["shutdown_on_close"] is False, ( - "the executor is a guest — closing it must not cull the run's cluster" - ) - closer() - assert record.get("client_closed") is True - assert record.get("cluster_closed") is True - def test_job_exec_prefix_cds_into_workdir() -> None: """Gateway worker pods start in the image's WORKDIR, not the diff --git a/tests/test_environment.py b/tests/test_environment.py new file mode 100644 index 00000000..2bc30246 --- /dev/null +++ b/tests/test_environment.py @@ -0,0 +1,293 @@ +"""Tests for the environment model: mode, env_version, lock scan.""" +from __future__ import annotations + +from pathlib import Path + +import pytest +from conftest import make_project + +from lightcone.engine.environment import ( + InstallSettings, + LockScan, + Mode, + ProjectEnvironmentError, + compute_env_version, + load_environment, + scan_lock, +) +from lightcone.engine.image.declaration import load_image_declaration + +# ---- mode detection -------------------------------------------------------- + + +class TestModeDetection: + def test_direct_by_default(self, direct_project: Path) -> None: + env = load_environment(direct_project) + assert env.mode is Mode.DIRECT + assert env.image is None + + def test_image_table_escalates(self, containerized_project: Path) -> None: + env = load_environment(containerized_project) + assert env.mode is Mode.CONTAINERIZED + assert env.image is not None + assert env.image.system_packages == ("libhdf5-dev", "r-base-core") + + def test_empty_image_table_escalates(self, tmp_path: Path) -> None: + """Presence IS the escalation — even an empty table.""" + project = make_project( + tmp_path / "p", extra_pyproject="\n[tool.lightcone.image]\n" + ) + assert load_environment(project).mode is Mode.CONTAINERIZED + + def test_extra_file_alone_escalates(self, direct_project: Path) -> None: + (direct_project / "Containerfile.extra").write_text("RUN echo hi\n") + env = load_environment(direct_project) + assert env.mode is Mode.CONTAINERIZED + assert env.image is not None and env.image.extra is not None + + def test_python_version_read(self, direct_project: Path) -> None: + assert load_environment(direct_project).python_version == "3.12.12" + + +class TestRefusals: + def test_authored_containerfile_refused(self, direct_project: Path) -> None: + (direct_project / "Containerfile").write_text("FROM debian\n") + with pytest.raises(ProjectEnvironmentError, match="generates images"): + load_environment(direct_project) + + def test_packaged_containerized_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + containerized=True, + extra_pyproject=( + '\n[build-system]\nrequires = ["hatchling"]\n' + 'build-backend = "hatchling.build"\n' + ), + ) + with pytest.raises(ProjectEnvironmentError, match="virtual project"): + load_environment(project) + + def test_packaged_direct_is_fine(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject=( + '\n[build-system]\nrequires = ["hatchling"]\n' + 'build-backend = "hatchling.build"\n' + ), + ) + assert load_environment(project).mode is Mode.DIRECT + + def test_missing_pyproject(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "p") + (project / "pyproject.toml").unlink() + with pytest.raises(ProjectEnvironmentError, match="pyproject.toml"): + load_environment(project) + + def test_missing_lock(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "p") + (project / "uv.lock").unlink() + with pytest.raises(ProjectEnvironmentError, match="uv lock"): + load_environment(project) + + def test_missing_python_version(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "p") + (project / ".python-version").unlink() + with pytest.raises(ProjectEnvironmentError, match=".python-version"): + load_environment(project) + + def test_unknown_lightcone_key_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", extra_pyproject="\n[tool.lightcone.bogus]\nx = 1\n" + ) + with pytest.raises(ProjectEnvironmentError, match="bogus"): + load_environment(project) + + +class TestWritableProject: + def test_parsed(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject=( + '\n[tool.lightcone.sandbox]\nwritable-project = ["result"]\n' + ), + ) + env = load_environment(project) + assert env.writable_project_outputs == frozenset({"result"}) + + def test_default_empty(self, direct_project: Path) -> None: + assert load_environment(direct_project).writable_project_outputs == frozenset() + + def test_bad_type_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject="\n[tool.lightcone.sandbox]\nwritable-project = 1\n", + ) + with pytest.raises(ProjectEnvironmentError, match="writable-project"): + load_environment(project) + + def test_unknown_sandbox_key_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject="\n[tool.lightcone.sandbox]\nbogus = 1\n", + ) + with pytest.raises(ProjectEnvironmentError, match="bogus"): + load_environment(project) + + +# ---- env_version ----------------------------------------------------------- + + +def _env_version_of(project: Path) -> str: + return load_environment(project).env_version + + +class TestEnvVersion: + def test_format(self, direct_project: Path) -> None: + ev = _env_version_of(direct_project) + assert ev.startswith("sha256:") and len(ev) == 7 + 64 + + def test_deterministic(self, tmp_path: Path) -> None: + a = _env_version_of(make_project(tmp_path / "a")) + b = _env_version_of(make_project(tmp_path / "b")) + assert a == b + + def test_golden_pinned(self, direct_project: Path) -> None: + """Golden fingerprint: moves ⇔ the identity formula changed. + + A failure here means every existing manifest in the wild goes + stale — deliberate formula changes must bump this string + consciously, in the same commit. + """ + assert _env_version_of(direct_project) == ( + "sha256:fe986a87d3d2e4f38e31e38a8930f5b6c425adc4199c7ba17fe875c6c2b37a81" + ) + + def test_golden_pinned_containerized( + self, containerized_project: Path + ) -> None: + assert _env_version_of(containerized_project) == ( + "sha256:12f886a8ada4cbddaaa18595056eeefc26fdbad5d7f4d358620574457d8d006e" + ) + + def test_moves_with_lock(self, direct_project: Path) -> None: + before = _env_version_of(direct_project) + (direct_project / "uv.lock").write_text( + (direct_project / "uv.lock").read_text() + "\n# drift\n" + ) + assert _env_version_of(direct_project) != before + + def test_moves_with_python_pin(self, direct_project: Path) -> None: + before = _env_version_of(direct_project) + (direct_project / ".python-version").write_text("3.12.11\n") + assert _env_version_of(direct_project) != before + + def test_moves_with_install_settings(self, tmp_path: Path) -> None: + plain = _env_version_of(make_project(tmp_path / "a")) + tweaked = _env_version_of( + make_project( + tmp_path / "b", extra_pyproject="\n[tool.uv]\nno-binary = true\n" + ) + ) + assert plain != tweaked + + def test_moves_with_image_declaration( + self, tmp_path: Path + ) -> None: + direct = _env_version_of(make_project(tmp_path / "a")) + containerized = _env_version_of( + make_project(tmp_path / "b", containerized=True) + ) + assert direct != containerized + + def test_moves_with_extra_stage(self, tmp_path: Path) -> None: + a = make_project(tmp_path / "a", containerized=True) + before = _env_version_of(a) + (a / "Containerfile.extra").write_text("RUN echo hi\n") + assert _env_version_of(a) != before + + def test_does_not_move_with_project_code(self, direct_project: Path) -> None: + """G5: code edits never move the environment identity.""" + before = _env_version_of(direct_project) + (direct_project / "analysis.py").write_text("x = 1\n") + (direct_project / "astra.yaml").write_text("outputs: []\n") + assert _env_version_of(direct_project) == before + + def test_one_formula_direct_hashes_empty_image(self) -> None: + """Direct mode hashes the empty image shape — same formula.""" + ev = compute_env_version( + uv_lock_bytes=b"lock", + python_version_bytes=b"3.12.12\n", + install_settings=InstallSettings.from_tool_uv({}), + image=None, + ) + assert ev.startswith("sha256:") + + +# ---- lock scan ------------------------------------------------------------- + + +_LOCK_WITH_HAZARDS = """\ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "fixture-proj" +version = "0.0.0" +source = { virtual = "." } + +[[package]] +name = "numpy" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +wheels = [{ url = "https://example/numpy.whl", hash = "sha256:aa" }] + +[[package]] +name = "legacy-sdist" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://example/legacy.tar.gz", hash = "sha256:bb" } + +[[package]] +name = "local-thing" +version = "0.1" +source = { directory = "../local-thing" } +""" + + +class TestScanLock: + def test_clean_lock(self, direct_project: Path) -> None: + scan = scan_lock(direct_project) + assert scan == LockScan(refusals=(), sdist_built=(), non_default_groups=()) + + def test_hazards(self, direct_project: Path) -> None: + (direct_project / "uv.lock").write_text(_LOCK_WITH_HAZARDS) + scan = scan_lock(direct_project) + assert scan.sdist_built == ("legacy-sdist",) + assert len(scan.refusals) == 1 and "local-thing" in scan.refusals[0] + + def test_own_package_exempt(self, direct_project: Path) -> None: + """The project's own (virtual/editable) package never refuses.""" + (direct_project / "uv.lock").write_text(_LOCK_WITH_HAZARDS) + scan = scan_lock(direct_project) + assert not any("fixture-proj" in r for r in scan.refusals) + + def test_group_advisory(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject='\n[dependency-groups]\ndocs = ["sphinx"]\n', + ) + assert scan_lock(project).non_default_groups == ("docs",) + + def test_default_groups_not_advisory(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject='\n[dependency-groups]\ndev = ["pytest"]\n', + ) + assert scan_lock(project).non_default_groups == () + + +# ---- declaration convenience ---------------------------------------------- + + +def test_load_image_declaration_none_for_direct(direct_project: Path) -> None: + assert load_image_declaration(direct_project) is None diff --git a/tests/test_image_builder.py b/tests/test_image_builder.py new file mode 100644 index 00000000..2d952886 --- /dev/null +++ b/tests/test_image_builder.py @@ -0,0 +1,254 @@ +"""Tests for the build record, context staging, and the podman builder +(mocked subprocess; the real-build smoke lives in test_image_smoke.py).""" +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from conftest import make_project + +from lightcone.engine.environment import load_environment +from lightcone.engine.image import ensure_image, image_status, resolve_pinned +from lightcone.engine.image.builder import BuildContext, BuildResult +from lightcone.engine.image.builder_podman import PodmanBuilder +from lightcone.engine.image.errors import ( + AptPackageNotFoundError, + BaseContractError, + ImageBuildError, + ImageMissingError, + PodmanUnavailableError, +) # noqa: F401 (PodmanUnavailableError used in patch-based tests) +from lightcone.engine.image.record import ( + BuildRecord, + read_record, + write_record, +) + +_RECORD = BuildRecord( + tag="lc-env-0123456789abcdef", + image_id="sha256:" + "aa" * 32, + digest="sha256:" + "bb" * 32, + platform="linux/amd64", + env_version="sha256:" + "cc" * 32, + lc_version="0.0.0", + base="docker.io/library/debian:bookworm-slim@sha256:" + "dd" * 32, + built_at="2026-08-17T00:00:00+00:00", + dpkg_snapshot_sha256="ee" * 32, +) + + +class TestRecord: + def test_round_trip(self, tmp_path: Path) -> None: + write_record(tmp_path, _RECORD, "ii libc6 2.36\n") + assert read_record(tmp_path) == _RECORD + snapshot = tmp_path / ".lightcone/image" / f"dpkg-snapshot-{_RECORD.tag}.txt" + assert snapshot.read_text() == "ii libc6 2.36\n" + + def test_missing_returns_none(self, tmp_path: Path) -> None: + assert read_record(tmp_path) is None + + def test_corrupt_returns_none(self, tmp_path: Path) -> None: + d = tmp_path / ".lightcone/image" + d.mkdir(parents=True) + (d / "record.json").write_text("not json") + assert read_record(tmp_path) is None + + +class TestBuildContext: + def test_stages_exactly_three_files(self, tmp_path: Path) -> None: + """G5 structural guarantee: no code path admits a fourth file — + and the staged bytes are exactly the bytes the tag hashed.""" + from lightcone.engine.image.identity import EnvInputs + + project = make_project(tmp_path / "proj") + ctx = BuildContext( + containerfile_text="FROM x\n", inputs=EnvInputs.read(project) + ) + staged = tmp_path / "staged" + containerfile = ctx.stage(staged) + assert sorted(p.name for p in staged.iterdir()) == [ + "Containerfile", "pyproject.toml", "uv.lock", + ] + assert containerfile.read_text() == "FROM x\n" + # World-readable (rootless build under a userns). + for p in staged.iterdir(): + assert p.stat().st_mode & 0o444 == 0o444 + + +class _FakeBuilder: + def __init__(self, *, exists: bool = False) -> None: + self._exists = exists + self.builds: list[str] = [] + + def exists(self, tag: str) -> bool: + return self._exists + + def build(self, context: BuildContext, *, tag: str) -> BuildResult: + self.builds.append(tag) + self._exists = True + return BuildResult( + tag=tag, + image_id="sha256:" + "aa" * 32, + digest=None, + platform="linux/amd64", + dpkg_snapshot_text="ii libc6\n", + ) + + +class TestEnsureImage: + def test_builds_and_records(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + builder = _FakeBuilder() + messages: list[str] = [] + record = ensure_image( + project, env, builder=builder, on_progress=messages.append + ) + assert builder.builds == [record.tag] + assert record.tag.startswith("lc-env-") + assert record.env_version == env.env_version + assert read_record(project) == record + assert any("~minutes" in m for m in messages) + + def test_tag_hit_is_noop(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + builder = _FakeBuilder() + first = ensure_image(project, env, builder=builder) + second = ensure_image(project, env, builder=builder) + assert builder.builds == [first.tag] # exactly one build + assert second == first + + def test_env_edit_rebuilds(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + builder = _FakeBuilder() + env = load_environment(project) + ensure_image(project, env, builder=builder) + (project / "uv.lock").write_text( + (project / "uv.lock").read_text() + "# drift\n" + ) + env2 = load_environment(project) + record2 = ensure_image(project, env2, builder=builder) + assert len(builder.builds) == 2 + assert builder.builds[1] == record2.tag != builder.builds[0] + + def test_force_rebuilds(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + builder = _FakeBuilder() + ensure_image(project, env, builder=builder) + ensure_image(project, env, builder=builder, force=True) + assert len(builder.builds) == 2 + + +class TestResolvePinned: + def test_missing_image_names_lc_build(self, tmp_path: Path) -> None: + """`lc run` never builds — the error embeds the exact command.""" + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + with pytest.raises(ImageMissingError, match="lc build"): + resolve_pinned(project, env, builder=_FakeBuilder()) + + def test_resolves_after_build(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + builder = _FakeBuilder() + record = ensure_image(project, env, builder=builder) + assert resolve_pinned(project, env, builder=builder) == record + + def test_stale_record_after_env_edit_errors(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + builder = _FakeBuilder() + ensure_image(project, env, builder=builder) + (project / "uv.lock").write_text( + (project / "uv.lock").read_text() + "# drift\n" + ) + with pytest.raises(ImageMissingError): + resolve_pinned(project, load_environment(project), builder=builder) + + +class TestImageStatus: + def test_needs_build_then_built(self, tmp_path: Path) -> None: + project = make_project(tmp_path / "proj", containerized=True) + env = load_environment(project) + builder = _FakeBuilder() + s = image_status(project, env, builder=builder) + assert not s.built and s.tag.startswith("lc-env-") + ensure_image(project, env, builder=builder) + s2 = image_status(project, env, builder=builder) + assert s2.built and s2.image_id is not None + + +class TestPodmanBuilderErrors: + def _builder(self) -> PodmanBuilder: + with patch("shutil.which", return_value="/usr/bin/podman"): + return PodmanBuilder() + + def test_missing_podman(self) -> None: + with patch("shutil.which", return_value=None): + with pytest.raises(PodmanUnavailableError, match="podman.io"): + PodmanBuilder() + + def test_apt_not_found_mapped(self) -> None: + b = self._builder() + with pytest.raises(AptPackageNotFoundError, match="apt-cache search rbase"): + b._raise_mapped("", "E: Unable to locate package rbase\nexit code: 100") + + def test_contract_exit_codes_mapped(self) -> None: + b = self._builder() + with pytest.raises(BaseContractError, match="musl"): + b._raise_mapped("", "…while running runtime: exit status 43") + with pytest.raises(BaseContractError, match="Containerfile.extra"): + b._raise_mapped("", "exit status 44") + + def test_arch_miss_mapped(self) -> None: + b = self._builder() + with pytest.raises(BaseContractError, match="linux/amd64"): + b._raise_mapped( + "", "no image found in manifest list for architecture arm64" + ) + + def test_generic_failure_bounded_tail(self) -> None: + b = self._builder() + noise = "\n".join(f"line {i}" for i in range(500)) + with pytest.raises(ImageBuildError) as exc: + b._raise_mapped(noise, "") + assert "line 499" in str(exc.value) + assert "line 0" not in str(exc.value) + + def test_build_argv(self, tmp_path: Path) -> None: + from lightcone.engine.image.identity import EnvInputs + + b = self._builder() + project = make_project(tmp_path / "proj") + ctx = BuildContext( + containerfile_text="FROM scratch\n", inputs=EnvInputs.read(project) + ) + calls: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + calls.append(list(cmd)) + if cmd[1] == "build": + return MagicMock(returncode=0, stdout="", stderr="") + if cmd[1] == "image": + return MagicMock( + returncode=0, + stdout="sha256:aa|sha256:bb|linux/amd64\n", + stderr="", + ) + return MagicMock(returncode=0, stdout="ii libc6\n", stderr="") + + with patch( + "lightcone.engine.image.builder_podman.subprocess.run", + side_effect=fake_run, + ): + result = b.build(ctx, tag="lc-env-abc") + build_cmd = calls[0] + assert build_cmd[0:2] == ["podman", "build"] + assert "--tag" in build_cmd and "lc-env-abc" in build_cmd + # Snapshot capture runs offline with the entrypoint cleared. + snap_cmd = calls[2] + assert "--net=none" in snap_cmd and "--entrypoint=" in snap_cmd + assert result.image_id == "sha256:aa" diff --git a/tests/test_image_declaration.py b/tests/test_image_declaration.py new file mode 100644 index 00000000..983a87be --- /dev/null +++ b/tests/test_image_declaration.py @@ -0,0 +1,116 @@ +"""Tests for the [tool.lightcone.image] declaration surface.""" +from __future__ import annotations + +from pathlib import Path + +import pytest +from conftest import make_project + +from lightcone.engine.image.declaration import ( + EMPTY_CANONICAL_JSON, + BaseRef, + load_image_declaration, +) +from lightcone.engine.image.errors import DeclarationError + +_DIGEST = "sha256:" + "9f" * 32 + + +class TestBaseRef: + def test_parse_digest_pinned(self) -> None: + ref = BaseRef.parse(f"nvcr.io/nvidia/cuda:12.4.1-runtime@{_DIGEST}") + assert ref.name == "nvcr.io/nvidia/cuda:12.4.1-runtime" + assert ref.digest == _DIGEST + assert str(ref) == f"nvcr.io/nvidia/cuda:12.4.1-runtime@{_DIGEST}" + + def test_tag_only_refused(self) -> None: + with pytest.raises(DeclarationError, match="pin the digest"): + BaseRef.parse("nvcr.io/nvidia/cuda:12.4.1-runtime") + + def test_bad_digest_refused(self) -> None: + with pytest.raises(DeclarationError, match="not.*valid digest"): + BaseRef.parse("debian:bookworm@sha256:nothex") + + def test_short_digest_refused(self) -> None: + with pytest.raises(DeclarationError, match="not.*valid digest"): + BaseRef.parse("debian:bookworm@sha256:abcd") + + +class TestLoadImageDeclaration: + def test_none_without_table(self, direct_project: Path) -> None: + assert load_image_declaration(direct_project) is None + + def test_packages_sorted_and_deduped(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject=( + "\n[tool.lightcone.image]\n" + 'system-packages = ["zlib1g", "r-base-core", "zlib1g"]\n' + ), + ) + decl = load_image_declaration(project) + assert decl is not None + assert decl.system_packages == ("r-base-core", "zlib1g") + + def test_unknown_key_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject='\n[tool.lightcone.image]\npackages = ["r"]\n', + ) + with pytest.raises(DeclarationError, match="unknown key"): + load_image_declaration(project) + + def test_bad_apt_name_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject=( + '\n[tool.lightcone.image]\nsystem-packages = ["R Base!"]\n' + ), + ) + with pytest.raises(DeclarationError, match="not a valid apt package name"): + load_image_declaration(project) + + def test_non_list_packages_refused(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject='\n[tool.lightcone.image]\nsystem-packages = "r"\n', + ) + with pytest.raises(DeclarationError, match="list of"): + load_image_declaration(project) + + def test_base_parsed(self, tmp_path: Path) -> None: + project = make_project( + tmp_path / "p", + extra_pyproject=( + f'\n[tool.lightcone.image]\nbase = "docker.io/x/y:1@{_DIGEST}"\n' + ), + ) + decl = load_image_declaration(project) + assert decl is not None and decl.base is not None + assert decl.base.digest == _DIGEST + + def test_extra_from_refused(self, direct_project: Path) -> None: + (direct_project / "Containerfile.extra").write_text( + "FROM debian:bookworm\nRUN echo hi\n" + ) + with pytest.raises(DeclarationError, match="FROM"): + load_image_declaration(direct_project) + + def test_extra_content_and_sha(self, direct_project: Path) -> None: + (direct_project / "Containerfile.extra").write_text("RUN echo hi\n") + decl = load_image_declaration(direct_project) + assert decl is not None + assert decl.extra == "RUN echo hi\n" + assert decl.extra_sha256 is not None and len(decl.extra_sha256) == 64 + + def test_canonical_json_stable(self, containerized_project: Path) -> None: + decl = load_image_declaration(containerized_project) + assert decl is not None + assert decl.canonical_json() == ( + '{"base":null,"system-packages":["libhdf5-dev","r-base-core"]}' + ) + + def test_empty_canonical_shape_matches(self) -> None: + """The direct-mode empty shape and a declaration's shape share + the same key structure — one env_version formula.""" + assert EMPTY_CANONICAL_JSON == '{"base":null,"system-packages":[]}' diff --git a/tests/test_image_identity.py b/tests/test_image_identity.py new file mode 100644 index 00000000..e778f465 --- /dev/null +++ b/tests/test_image_identity.py @@ -0,0 +1,74 @@ +"""Tests for the content-addressed image tag.""" +from __future__ import annotations + +import re +from pathlib import Path + +from conftest import make_project + +from lightcone.engine.environment import load_environment +from lightcone.engine.image.definition import ImageDefinition +from lightcone.engine.image.identity import EnvInputs, compute_tag +from lightcone.engine.image.render import render + + +def _tag_of(project: Path) -> str: + env = load_environment(project) + assert env.image is not None + defn = ImageDefinition.from_declaration( + env.image, + env_version=env.env_version, + python_version=env.python_version, + ) + return compute_tag(render(defn), EnvInputs.read(project)) + + +class TestTag: + def test_format(self, containerized_project: Path) -> None: + assert re.fullmatch(r"lc-env-[0-9a-f]{16}", _tag_of(containerized_project)) + + def test_deterministic(self, tmp_path: Path) -> None: + a = _tag_of(make_project(tmp_path / "a", containerized=True)) + b = _tag_of(make_project(tmp_path / "b", containerized=True)) + assert a == b + + def test_moves_with_package_add(self, tmp_path: Path) -> None: + a = _tag_of(make_project(tmp_path / "a", containerized=True)) + b_proj = make_project(tmp_path / "b") + (b_proj / "pyproject.toml").write_text( + (b_proj / "pyproject.toml").read_text() + + '\n[tool.lightcone.image]\nsystem-packages = ["bc"]\n' + ) + assert _tag_of(b_proj) != a + + def test_moves_with_lock_byte(self, containerized_project: Path) -> None: + before = _tag_of(containerized_project) + (containerized_project / "uv.lock").write_text( + (containerized_project / "uv.lock").read_text() + "# x\n" + ) + assert _tag_of(containerized_project) != before + + def test_moves_with_pyproject_byte(self, containerized_project: Path) -> None: + before = _tag_of(containerized_project) + p = containerized_project / "pyproject.toml" + p.write_text(p.read_text() + "# comment\n") + assert _tag_of(containerized_project) != before + + def test_moves_with_python_pin(self, containerized_project: Path) -> None: + before = _tag_of(containerized_project) + (containerized_project / ".python-version").write_text("3.12.11\n") + assert _tag_of(containerized_project) != before + + def test_moves_with_extra_stage(self, containerized_project: Path) -> None: + before = _tag_of(containerized_project) + (containerized_project / "Containerfile.extra").write_text("RUN echo x\n") + assert _tag_of(containerized_project) != before + + def test_does_not_move_with_project_code( + self, containerized_project: Path + ) -> None: + """G5: code edits change no input to the tag.""" + before = _tag_of(containerized_project) + (containerized_project / "analysis.py").write_text("x = 1\n") + (containerized_project / "astra.yaml").write_text("outputs: []\n") + assert _tag_of(containerized_project) == before diff --git a/tests/test_image_render.py b/tests/test_image_render.py new file mode 100644 index 00000000..1b182c86 --- /dev/null +++ b/tests/test_image_render.py @@ -0,0 +1,156 @@ +"""Golden tests for the generated Containerfile. + +The rendered text is half of the image tag's identity — these goldens +pin it byte-for-byte. Regenerate deliberately with: + + uv run python -m pytest tests/test_image_render.py --regen-goldens +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from lightcone.engine.image import constants +from lightcone.engine.image.declaration import BaseRef, ImageDeclaration +from lightcone.engine.image.definition import ImageDefinition +from lightcone.engine.image.render import render + +GOLDENS = Path(__file__).parent / "goldens" + +_ENV_VERSION = "sha256:" + "ab" * 32 +_BASE = BaseRef.parse( + "nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:" + "9f" * 32 +) + + +def _decl( + *, + base: BaseRef | None = None, + packages: tuple[str, ...] = (), + extra: str | None = None, +) -> ImageDeclaration: + import hashlib + + return ImageDeclaration( + base=base, + system_packages=packages, + extra=extra, + extra_sha256=( + hashlib.sha256(extra.encode()).hexdigest() if extra else None + ), + ) + + +def _definition(decl: ImageDeclaration) -> ImageDefinition: + from lightcone.engine.image.definition import DEFAULT_BASE, UV_DIST + + return ImageDefinition( + base=decl.base or DEFAULT_BASE, + system_packages=decl.system_packages, + python_version="3.12.12", + uv=UV_DIST, + extra_stage=decl.extra, + env_version=_ENV_VERSION, + ) + + +CASES = { + "minimal": _decl(), + "packages": _decl(packages=("libhdf5-dev", "r-base-core")), + "custom-base": _decl(base=_BASE, packages=("texlive-latex-base",)), + "extra-stage": _decl( + packages=("r-base-core",), + extra="RUN Rscript -e 'install.packages(\"cmdstanr\")'\n", + ), +} + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "case_name" in metafunc.fixturenames: + metafunc.parametrize("case_name", sorted(CASES)) + + +class TestGolden: + def test_matches_golden(self, case_name: str, request: pytest.FixtureRequest) -> None: + rendered = render(_definition(CASES[case_name])).text + golden = GOLDENS / f"{case_name}.Containerfile" + if request.config.getoption("--regen-goldens"): + golden.parent.mkdir(exist_ok=True) + golden.write_text(rendered) + pytest.skip("golden regenerated") + assert golden.is_file(), f"missing golden {golden}; run --regen-goldens" + assert rendered == golden.read_text() + + +class TestInvariants: + def test_deterministic(self) -> None: + d = _definition(CASES["packages"]) + assert render(d).text == render(d).text + + def test_package_order_is_canonical(self) -> None: + """Shuffled declaration input renders identically (packages are + sorted into the declaration, not at render time — but pin the + composed behaviour end to end).""" + a = render(_definition(_decl(packages=("a-pkg", "b-pkg")))).text + b = render(_definition(_decl(packages=("a-pkg", "b-pkg")))).text + assert a == b + + def test_offline_env_only_in_final_stage(self) -> None: + """THE ordering invariant (spec §11 step 6): no offline key may + appear textually before the final stage — the build's own sync + layer must keep network.""" + for case in CASES.values(): + text = render(_definition(case)).text + final_at = text.index("AS final") + for key in constants.OFFLINE_ENV: + assert key not in text[:final_at], ( + f"{key} leaked above the final stage" + ) + assert key in text[final_at:] + + def test_apt_layer_iff_packages(self) -> None: + assert "apt-get install" not in render(_definition(CASES["minimal"])).text + assert "apt-get install" in render(_definition(CASES["packages"])).text + + def test_apt_contract_check_iff_packages(self) -> None: + assert ( + f"exit {constants.EXIT_NO_APT}" + not in render(_definition(CASES["minimal"])).text + ) + assert ( + f"exit {constants.EXIT_NO_APT}" + in render(_definition(CASES["packages"])).text + ) + + def test_snapshot_after_extra_stage(self) -> None: + """The dpkg snapshot runs in the final stage, after the extra + stage — packages an extra stage installs are attested too.""" + text = render(_definition(CASES["extra-stage"])).text + assert text.index("FROM env AS extra") < text.index( + constants.DPKG_SNAPSHOT_PATH + ) + assert "FROM extra AS final" in text + + def test_no_extra_stage_finals_from_env(self) -> None: + assert "FROM env AS final" in render(_definition(CASES["minimal"])).text + + def test_no_project_code_enters_context(self) -> None: + """G5 structural check: the only COPY from the build context is + the two environment files.""" + for case in CASES.values(): + text = render(_definition(case)).text + copies = [ + line + for line in text.splitlines() + if line.startswith("COPY") and "--from=" not in line + ] + assert copies == ["COPY pyproject.toml uv.lock ./"] + + def test_sync_flags(self) -> None: + text = render(_definition(CASES["minimal"])).text + assert "--locked --exact --no-install-project --compile-bytecode" in text + + def test_env_version_label(self) -> None: + text = render(_definition(CASES["minimal"])).text + assert f'LABEL io.lightcone.env-version="{_ENV_VERSION}"' in text diff --git a/tests/test_image_runtime.py b/tests/test_image_runtime.py new file mode 100644 index 00000000..164e002e --- /dev/null +++ b/tests/test_image_runtime.py @@ -0,0 +1,165 @@ +"""Unit tests for the mount set, podman run argv, and machine preflight.""" +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from lightcone.engine.image import machine as machine_mod +from lightcone.engine.image.errors import DeclarationError, MachinePreflightError +from lightcone.engine.image.mounts import MountSet, compute_mount_set +from lightcone.engine.image.record import BuildRecord +from lightcone.engine.image.runtime_podman import PodmanRuntime + +_RECORD = BuildRecord( + tag="lc-env-0123456789abcdef", + image_id="sha256:" + "aa" * 32, + digest=None, + platform="linux/amd64", + env_version="sha256:" + "cc" * 32, + lc_version="0", + base="docker.io/library/debian:bookworm-slim@sha256:" + "dd" * 32, + built_at="2026-08-17T00:00:00+00:00", + dpkg_snapshot_sha256="ee" * 32, +) + + +class TestMountSet: + def test_project_rw_inputs_ro(self, tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + data = tmp_path / "data" / "cat.fits" + data.parent.mkdir() + data.write_text("x") + ms = compute_mount_set(project, external_inputs=[data]) + args = ms.to_podman_args() + assert f"{project.resolve()}:{project.resolve()}:rw" in args + assert f"{data.resolve()}:{data.resolve()}:ro" in args + + def test_probe_mounts_project_ro(self, tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + ms = compute_mount_set(project, readonly_project=True) + assert f"{project.resolve()}:{project.resolve()}:ro" in ms.to_podman_args() + + def test_in_tree_inputs_deduped(self, tmp_path: Path) -> None: + project = tmp_path / "proj" + (project / "data").mkdir(parents=True) + (project / "data" / "x.txt").write_text("x") + ms = compute_mount_set( + project, external_inputs=[project / "data" / "x.txt"] + ) + assert ms.external_inputs == () + + def test_nested_external_inputs_deduped(self, tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + outer = tmp_path / "catalogs" + (outer / "sub").mkdir(parents=True) + (outer / "sub" / "x.txt").write_text("x") + ms = compute_mount_set( + project, external_inputs=[outer / "sub" / "x.txt", outer] + ) + assert ms.external_inputs == (outer.resolve(),) + + def test_parent_of_project_refused(self, tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + with pytest.raises(DeclarationError, match="widen"): + compute_mount_set(project, external_inputs=[tmp_path]) + + def test_tmpfs_and_shm(self, tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + args = compute_mount_set(project).to_podman_args() + assert "/tmp:rw,exec" in args + assert "--shm-size" in args + + +class TestRunArgv: + def _runtime(self) -> PodmanRuntime: + with patch("shutil.which", return_value="/usr/bin/podman"): + return PodmanRuntime() + + def _argv(self, tmp_path: Path, **kwargs) -> list[str]: # type: ignore[no-untyped-def] + project = tmp_path / "proj" + project.mkdir(exist_ok=True) + return self._runtime().run_argv( + record=_RECORD, + mounts=MountSet(project=project.resolve(), external_inputs=()), + argv=["/opt/venv/bin/lc", "materialize"], + **kwargs, + ) + + def test_isolation_flags(self, tmp_path: Path) -> None: + argv = self._argv(tmp_path) + for flag in ("--net=none", "--userns=keep-id", "--entrypoint=", "--pull=never"): + assert flag in argv + assert "label=disable" in argv + + def test_pinned_by_image_id(self, tmp_path: Path) -> None: + """The pin point: the image reference in the argv is the recorded + image id, not the tag — a retagged image cannot substitute.""" + argv = self._argv(tmp_path) + assert _RECORD.image_id in argv + assert _RECORD.tag not in argv[argv.index(_RECORD.image_id):] + + def test_identity_env_injected(self, tmp_path: Path) -> None: + argv = self._argv(tmp_path) + assert "LC_DELEGATED=1" in argv + assert "LC_WORKER_RUNTIME=container" in argv + assert "LC_CONTAINER_NETWORK=none" in argv + assert f"LC_IMAGE_DIGEST={_RECORD.image_id}" in argv + + def test_env_is_allowlist_not_ambient( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "hunter2") + monkeypatch.setenv("TERM", "xterm-256color") + argv = self._argv(tmp_path) + joined = " ".join(argv) + assert "hunter2" not in joined + assert "TERM=xterm-256color" in joined + + def test_command_re_enters_lc(self, tmp_path: Path) -> None: + argv = self._argv(tmp_path) + assert argv[-2:] == ["/opt/venv/bin/lc", "materialize"] + + +class TestMachinePreflight: + def test_linux_noop(self) -> None: + machine_mod.machine_preflight([Path("/anywhere")]) + + def _darwin(self, monkeypatch: pytest.MonkeyPatch, inspect: dict | None) -> None: + monkeypatch.setattr(machine_mod.platform, "system", lambda: "Darwin") + monkeypatch.setattr(machine_mod, "_machine_inspect", lambda podman: inspect) + + def test_no_machine_refused(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._darwin(monkeypatch, None) + with pytest.raises(MachinePreflightError, match="podman machine init"): + machine_mod.machine_preflight([Path("/Users/x/proj")]) + + def test_stopped_machine_refused(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._darwin(monkeypatch, {"State": "stopped"}) + with pytest.raises(MachinePreflightError, match="podman machine start"): + machine_mod.machine_preflight([Path("/Users/x/proj")]) + + def test_unshared_source_refused(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A mount source outside the VM's shares is a refusal naming the + fix — never a silently empty mount.""" + self._darwin( + monkeypatch, + {"State": "running", "Mounts": [{"Source": "/Users"}]}, + ) + with pytest.raises( + MachinePreflightError, match="podman machine set --volume" + ): + machine_mod.machine_preflight([Path("/Volumes/scratch/data")]) + + def test_shared_sources_pass(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._darwin( + monkeypatch, + {"State": "running", "Mounts": [{"Source": "/Users"}]}, + ) + machine_mod.machine_preflight([Path("/Users/x/proj")]) diff --git a/tests/test_image_smoke.py b/tests/test_image_smoke.py new file mode 100644 index 00000000..1c50a606 --- /dev/null +++ b/tests/test_image_smoke.py @@ -0,0 +1,132 @@ +"""Real podman build smoke (opt-in: -m podman; needs network + rootless +podman).""" +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest +from conftest import make_project + +from lightcone.engine.environment import load_environment +from lightcone.engine.image import constants, ensure_image, read_record + +pytestmark = [ + pytest.mark.podman, + pytest.mark.slow, + pytest.mark.skipif(shutil.which("podman") is None, reason="podman not installed"), +] + + +@pytest.fixture(scope="module") +def built(tmp_path_factory: pytest.TempPathFactory): # type: ignore[no-untyped-def] + """Build the minimal containerized project image once per module.""" + project = make_project( + tmp_path_factory.mktemp("smoke") / "proj", + extra_pyproject='\n[tool.lightcone.image]\nsystem-packages = ["bc"]\n', + ) + env = load_environment(project) + record = ensure_image(project, env) + return project, env, record + + +class TestRealBuild: + def test_record_written(self, built) -> None: # type: ignore[no-untyped-def] + project, env, record = built + assert read_record(project) == record + assert record.env_version == env.env_version + assert record.image_id.startswith("sha256:") + + def test_tag_hit_is_noop(self, built) -> None: # type: ignore[no-untyped-def] + project, env, record = built + again = ensure_image(project, env) + assert again == record + + def test_baked_identity_and_env(self, built) -> None: # type: ignore[no-untyped-def] + project, env, record = built + out = subprocess.run( + [ + "podman", "run", "--rm", "--pull=never", "--net=none", + "--entrypoint=", record.image_id, + "cat", constants.IDENTITY_PATH, + ], + capture_output=True, + text=True, + check=True, + ) + identity = json.loads(out.stdout) + assert identity["env_version"] == env.env_version + assert identity["python_version"] == "3.12.12" + + def test_interpreter_and_venv_baked(self, built) -> None: # type: ignore[no-untyped-def] + _, _, record = built + out = subprocess.run( + [ + "podman", "run", "--rm", "--pull=never", "--net=none", + "--entrypoint=", record.image_id, + f"{constants.OPT_VENV}/bin/python", "-c", + "import sys; print(sys.version.split()[0])", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "3.12.12" + + def test_system_package_installed(self, built) -> None: # type: ignore[no-untyped-def] + _, _, record = built + out = subprocess.run( + [ + "podman", "run", "--rm", "--pull=never", "--net=none", + "--entrypoint=", record.image_id, + "sh", "-c", "echo '2+40' | bc", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "42" + + def test_offline_env_baked(self, built) -> None: # type: ignore[no-untyped-def] + _, _, record = built + out = subprocess.run( + [ + "podman", "run", "--rm", "--pull=never", "--net=none", + "--entrypoint=", record.image_id, + "sh", "-c", "echo $UV_OFFLINE $UV_PYTHON_DOWNLOADS", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.split() == ["1", "never"] + + def test_dpkg_snapshot_attests_bc(self, built) -> None: # type: ignore[no-untyped-def] + project, _, record = built + snapshot = ( + project / ".lightcone/image" / f"dpkg-snapshot-{record.tag}.txt" + ).read_text() + assert "bc" in snapshot + + +class TestAptErrorEndToEnd: + def test_unlocatable_package_pointed_error( + self, tmp_path: Path + ) -> None: + from lightcone.engine.image.errors import AptPackageNotFoundError + + project = make_project( + tmp_path / "proj", + extra_pyproject=( + "\n[tool.lightcone.image]\n" + 'system-packages = ["lc-no-such-package-zz"]\n' + ), + ) + env = load_environment(project) + with pytest.raises( + AptPackageNotFoundError, + match="no apt package named `lc-no-such-package-zz`", + ): + ensure_image(project, env) diff --git a/tests/test_launcher.py b/tests/test_launcher.py new file mode 100644 index 00000000..e1453ccb --- /dev/null +++ b/tests/test_launcher.py @@ -0,0 +1,187 @@ +"""Tests for the tool-env launcher (discover → scrub → converge → exec).""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from conftest import make_project + +from lightcone import launcher +from lightcone.launcher import TOOL_ENV_VERBS, maybe_delegate + + +@pytest.fixture +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + p = make_project(tmp_path / "proj") + monkeypatch.chdir(p) + return p + + +@pytest.fixture +def exec_calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, list[str], dict]]: + calls: list[tuple[str, list[str], dict]] = [] + + def fake_execve(path: str, argv: list[str], env: dict) -> None: + calls.append((path, argv, env)) + raise SystemExit(0) # exec never returns; emulate process handoff + + monkeypatch.setattr(os, "execve", fake_execve) + return calls + + +@pytest.fixture +def sync_calls(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + calls: list[list[str]] = [] + + def fake_converge(root: Path) -> None: + calls.append(["sync", str(root)]) + (root / ".venv" / "bin").mkdir(parents=True, exist_ok=True) + (root / ".venv" / "bin" / "lc").write_text("#!/bin/sh\n") + + monkeypatch.setattr(launcher, "_converge_direct", fake_converge) + return calls + + +class TestRouting: + def test_tool_env_verbs_never_delegate( + self, project: Path, exec_calls: list, sync_calls: list + ) -> None: + for verb in sorted(TOOL_ENV_VERBS): + maybe_delegate([verb]) + assert exec_calls == [] + assert sync_calls == [] + + def test_already_delegated_returns( + self, + project: Path, + exec_calls: list, + sync_calls: list, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("LC_DELEGATED", "1") + maybe_delegate(["materialize"]) + assert exec_calls == [] + + def test_no_args_returns(self, project: Path, exec_calls: list) -> None: + maybe_delegate([]) + maybe_delegate(["--help"]) + maybe_delegate(["--version"]) + assert exec_calls == [] + + def test_no_project_returns( + self, tmp_path: Path, exec_calls: list, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + maybe_delegate(["materialize"]) + assert exec_calls == [] + + +class TestDelegation: + def test_frozen_interface( + self, project: Path, exec_calls: list, sync_calls: list + ) -> None: + """THE frozen contract: exec of /bin/lc with verbatim argv + passthrough + LC_DELEGATED=1 — a tool-env launcher of any + version must be able to delegate to an engine of any age, so + nothing else may travel across this boundary.""" + with pytest.raises(SystemExit): + maybe_delegate(["materialize", "-u", "baseline", "best_fit"]) + assert len(exec_calls) == 1 + path, argv, env = exec_calls[0] + assert path == str(project / ".venv" / "bin" / "lc") + assert argv == ["lc", "materialize", "-u", "baseline", "best_fit"] + assert env["LC_DELEGATED"] == "1" + + def test_converges_before_exec( + self, project: Path, exec_calls: list, sync_calls: list + ) -> None: + with pytest.raises(SystemExit): + maybe_delegate(["run", "python", "-V"]) + assert sync_calls == [["sync", str(project)]] + + def test_scrubs_ambient_uv_env( + self, + project: Path, + exec_calls: list, + sync_calls: list, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Ambient UV_* steering must never reach the engine — a + UV_PROJECT pointing elsewhere would converge the wrong env.""" + monkeypatch.setenv("UV_PROJECT", "/somewhere/else") + monkeypatch.setenv("UV_INDEX_URL", "https://evil.example/simple") + with pytest.raises(SystemExit): + maybe_delegate(["materialize"]) + _, _, env = exec_calls[0] + assert "UV_PROJECT" not in env + assert "UV_INDEX_URL" not in env + + def test_missing_engine_after_sync_fails_loud( + self, + project: Path, + exec_calls: list, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + """Never a PATH fallback: a synced env without the engine binary + is a hard, explained error.""" + monkeypatch.setattr(launcher, "_converge_direct", lambda root: None) + with pytest.raises(SystemExit) as exc: + maybe_delegate(["materialize"]) + assert exc.value.code == 1 + assert "uv add lightcone-cli" in capsys.readouterr().err + assert exec_calls == [] + + def test_containerized_delegates_into_image( + self, + tmp_path: Path, + exec_calls: list, + sync_calls: list, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The containerized branch hands off to the podman full-stack + delegation — never a host sync, never a host .venv exec.""" + delegations: list[tuple[Path, list[str], str]] = [] + + def fake_delegate(root: Path, env, argv: list[str], verb: str) -> None: + delegations.append((root, argv, verb)) + raise SystemExit(0) + + monkeypatch.setattr(launcher, "_delegate_containerized", fake_delegate) + p = make_project(tmp_path / "proj", containerized=True) + monkeypatch.chdir(p) + with pytest.raises(SystemExit): + maybe_delegate(["materialize", "-u", "baseline"]) + assert delegations == [(p, ["materialize", "-u", "baseline"], "materialize")] + assert exec_calls == [] + assert sync_calls == [] + + def test_environment_error_is_clean( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + p = make_project(tmp_path / "proj") + (p / "uv.lock").unlink() + monkeypatch.chdir(p) + with pytest.raises(SystemExit) as exc: + maybe_delegate(["materialize"]) + assert exc.value.code == 1 + err = capsys.readouterr().err + assert "uv lock" in err + assert "Traceback" not in err + + +def test_tool_env_verbs_and_click_commands_agree() -> None: + """Parity pin: every Click command is either a tool-env verb or an + intentionally delegated one. A new command added to commands.py + without a routing decision here would delegate by default — into a + project-locked engine that may not know the verb, a confusing + failure far from its cause. (The launcher deliberately never + imports commands.py — this test is the link.)""" + from lightcone.cli.commands import main as click_main + + delegated = {"materialize", "run"} + assert set(click_main.commands) == TOOL_ENV_VERBS | delegated diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 73dce1c9..6c38b5b9 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -11,11 +11,14 @@ SCHEMA_VERSION, code_version, fingerprint_external, + is_pre_migration, read_manifest, sha256_dir, write_manifest, ) +_ENV = "sha256:" + "ee" * 32 + def _write(path: Path, content: bytes | str) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -136,39 +139,44 @@ def test_fingerprint_external_missing_returns_marker(tmp_path: Path) -> None: def test_code_version_deterministic() -> None: cv1 = code_version( recipe="python script.py --x 1", - container_image="lc-foo-abc123", decisions={"k": "a", "j": 1}, + env_version=_ENV, ) cv2 = code_version( recipe="python script.py --x 1", - container_image="lc-foo-abc123", decisions={"j": 1, "k": "a"}, + env_version=_ENV, ) assert cv1 == cv2 assert cv1.startswith("sha256:") def test_code_version_changes_on_recipe() -> None: - cv1 = code_version(recipe="a", container_image="c", decisions={}) - cv2 = code_version(recipe="b", container_image="c", decisions={}) + cv1 = code_version(recipe="a", decisions={}, env_version=_ENV) + cv2 = code_version(recipe="b", decisions={}, env_version=_ENV) assert cv1 != cv2 -def test_code_version_changes_on_container() -> None: - cv1 = code_version(recipe="r", container_image="c1", decisions={}) - cv2 = code_version(recipe="r", container_image="c2", decisions={}) +def test_code_version_changes_on_env_version() -> None: + cv1 = code_version(recipe="r", decisions={}, env_version=_ENV) + cv2 = code_version(recipe="r", decisions={}, env_version="sha256:" + "ff" * 32) assert cv1 != cv2 def test_code_version_changes_on_decisions() -> None: - cv1 = code_version(recipe="r", container_image="c", decisions={"k": 1}) - cv2 = code_version(recipe="r", container_image="c", decisions={"k": 2}) + cv1 = code_version(recipe="r", decisions={"k": 1}, env_version=_ENV) + cv2 = code_version(recipe="r", decisions={"k": 2}, env_version=_ENV) assert cv1 != cv2 -def test_code_version_handles_none_container() -> None: - cv = code_version(recipe="r", container_image=None, decisions={}) - assert cv.startswith("sha256:") +def test_code_version_changes_on_writable_project() -> None: + """The per-output sandbox escalation is materialization-relevant — + but per-output: it moves only this output's code_version.""" + cv1 = code_version(recipe="r", decisions={}, env_version=_ENV) + cv2 = code_version( + recipe="r", decisions={}, env_version=_ENV, writable_project=True + ) + assert cv1 != cv2 # ---- write_manifest ------------------------------------------------------- @@ -187,7 +195,7 @@ def test_write_manifest_basic(tmp_path: Path) -> None: "output_id": "foo", "universe_id": "u1", "recipe": "python script.py", - "container_image": "lc-foo-abc", + "env_version": _ENV, "decisions": {"k": 1}, "code_version": "sha256:abc", "git_sha": "deadbeef", @@ -204,7 +212,7 @@ def test_write_manifest_basic(tmp_path: Path) -> None: assert m["output_id"] == "foo" assert m["universe_id"] == "u1" assert m["recipe"] == "python script.py" - assert m["container_image"] == "lc-foo-abc" + assert m["env_version"] == _ENV assert m["decisions"] == {"k": 1} assert m["code_version"] == "sha256:abc" assert m["git_sha"] == "deadbeef" @@ -215,6 +223,14 @@ def test_write_manifest_basic(tmp_path: Path) -> None: assert m["input_versions"]["raw_data"].startswith("mtime-size:") assert "finished_at" in m assert "host" in m + assert m["worker_runtime"] == "host" + assert m["image"] is None + assert m["dpkg_snapshot_sha256"] is None + assert m["sdist_built"] == [] + # No enforcement ran and none was claimed — the honest default. + assert m["hermeticity"] == { + "mechanism": "none", "fs": "open", "network": "allowed", + } def test_write_manifest_chains_upstream_data_version(tmp_path: Path) -> None: @@ -230,9 +246,9 @@ def test_write_manifest_chains_upstream_data_version(tmp_path: Path) -> None: "output_id": "upstream", "universe_id": "u1", "recipe": "echo", - "container_image": None, "decisions": {}, "code_version": "sha256:up", + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -249,9 +265,9 @@ def test_write_manifest_chains_upstream_data_version(tmp_path: Path) -> None: "output_id": "downstream", "universe_id": "u1", "recipe": "echo", - "container_image": None, "decisions": {}, "code_version": "sha256:dn", + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -269,9 +285,9 @@ def test_write_manifest_atomic(tmp_path: Path) -> None: "output_id": "o", "universe_id": "u", "recipe": "r", - "container_image": None, "decisions": {}, "code_version": "sha256:c", + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", } @@ -293,9 +309,9 @@ def test_write_manifest_data_version_matches_sha256_dir(tmp_path: Path) -> None: "output_id": "x", "universe_id": "u", "recipe": "r", - "container_image": None, "decisions": {}, "code_version": "sha256:c", + "env_version": _ENV, "git_sha": "g", "lc_version": "0", }, @@ -317,9 +333,9 @@ def test_read_manifest_present(tmp_path: Path) -> None: "output_id": "o", "universe_id": "u", "recipe": "r", - "container_image": None, "decisions": {}, "code_version": "sha256:c", + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -363,28 +379,85 @@ def test_read_manifest_propagates_oserror(tmp_path: Path) -> None: manifest_path.chmod(0o644) -def test_manifest_records_worker_image_from_env( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """On a Gateway deployment the pod env carries the image it was - started with — the manifest records it as ground truth.""" + +# ---- schema v2 ------------------------------------------------------------ + + +def test_manifest_v2_field_list_golden(tmp_path: Path) -> None: + """The normative field enumeration (spec §3) — the single list the + SCHEMA_VERSION bump implements. A failure here means the schema + changed without a conscious decision.""" out = tmp_path / "out" - out.mkdir() - (out / "data.txt").write_text("x") - cfg = { - "output_id": "o", - "universe_id": "u", - "recipe": "echo", - "container_image": "Containerfile", - "decisions": {}, - "code_version": "sha256:x", - "git_sha": None, - "lc_version": "0", - } - monkeypatch.delenv("LIGHTCONE_WORKER_IMAGE", raising=False) - write_manifest(output_dir=out, inputs={}, cfg=cfg) - assert read_manifest(out)["worker_image"] is None + _write(out / "x", b"1") + write_manifest( + output_dir=out, + inputs={}, + cfg={ + "output_id": "o", "universe_id": "u", "recipe": "r", + "decisions": {}, "code_version": "sha256:c", "env_version": _ENV, + }, + attestation={ + "uv_version": "0.12.3", + "platform": {"os_release": "x", "kernel": "k", "glibc": "g", "arch": "a"}, + "python_build": "CPython 3.12.12", + "env_snapshot": {"locale": None, "tz": None}, + "gpu_driver": None, + }, + ) + m = json.loads((out / MANIFEST_FILENAME).read_text()) + assert sorted(m) == [ + "code_version", "data_version", "decisions", "dpkg_snapshot_sha256", + "env_snapshot", "env_version", "finished_at", "git_dirty", + "git_remote", "git_sha", "gpu_driver", "hermeticity", "host", + "image", "input_versions", "lc_version", "output_id", "platform", + "python_build", "recipe", "schema_version", "sdist_built", + "universe_id", "uv_version", "worker_runtime", + ] + + +def test_write_manifest_records_hermeticity(tmp_path: Path) -> None: + out = tmp_path / "out" + _write(out / "x", b"1") + write_manifest( + output_dir=out, + inputs={}, + cfg={ + "output_id": "o", "universe_id": "u", "recipe": "r", + "decisions": {}, "code_version": "sha256:c", "env_version": _ENV, + }, + hermeticity={ + "mechanism": "landlock", "fs": "declared", + "network": "unenforced", "landlock_abi": 9, + }, + ) + m = json.loads((out / MANIFEST_FILENAME).read_text()) + assert m["hermeticity"]["mechanism"] == "landlock" + assert m["hermeticity"]["landlock_abi"] == 9 - monkeypatch.setenv("LIGHTCONE_WORKER_IMAGE", "reg/lc-p:abc") - write_manifest(output_dir=out, inputs={}, cfg=cfg) - assert read_manifest(out)["worker_image"] == "reg/lc-p:abc" + +def test_write_manifest_records_image(tmp_path: Path) -> None: + out = tmp_path / "out" + _write(out / "x", b"1") + write_manifest( + output_dir=out, + inputs={}, + cfg={ + "output_id": "o", "universe_id": "u", "recipe": "r", + "decisions": {}, "code_version": "sha256:c", "env_version": _ENV, + "worker_runtime": "container", + "image_tag": "lc-env-abcd", "image_digest": "sha256:dd", + "dpkg_snapshot_sha256": "ee", + }, + ) + m = json.loads((out / MANIFEST_FILENAME).read_text()) + assert m["worker_runtime"] == "container" + assert m["image"] == {"tag": "lc-env-abcd", "digest": "sha256:dd"} + assert m["dpkg_snapshot_sha256"] == "ee" + + +def test_is_pre_migration() -> None: + assert is_pre_migration({"schema_version": 1, "code_version": "x"}) + assert is_pre_migration({}) + assert not is_pre_migration( + {"schema_version": SCHEMA_VERSION, "env_version": _ENV} + ) diff --git a/tests/test_runner.py b/tests/test_runner.py index c4e9ed0c..135a60da 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,18 +1,25 @@ -"""Tests for the per-rule run_rule helper. +"""Tests for the per-rule run_rule helper — the worker sequence. -The helper is invoked from the generated Snakefile's ``run:`` block; we -exercise it directly with a synthetic cfg, capturing stdout to assert on -the sentinel-prefixed framing the executor relies on. +run_rule is invoked from the generated Snakefile's ``run:`` block with +cwd == project root; we exercise it directly against fixture projects, +capturing stdout to assert on the sentinel-prefixed framing the +executor relies on. """ from __future__ import annotations import io +import json import re import subprocess from contextlib import redirect_stdout from pathlib import Path -from lightcone.engine.runner import SENTINEL, run_rule +import pytest +from conftest import make_project + +from lightcone.engine import runner +from lightcone.engine.environment import load_environment +from lightcone.engine.runner import SENTINEL, RuleGateError, run_rule _ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") @@ -32,30 +39,39 @@ def _capture(fn) -> tuple[str, BaseException | None]: return buf.getvalue(), err -def _cfg(output_id: str = "foo", *, shell_command: str = "echo hi") -> dict: - """Minimal cfg matching what the Snakefile generator writes. +@pytest.fixture +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + p = make_project(tmp_path / "proj") + monkeypatch.chdir(p) + return p + + +@pytest.fixture(autouse=True) +def _skip_uv_check(monkeypatch: pytest.MonkeyPatch) -> None: + """The env check shells out to `uv sync --check` against a real + project — fixture projects have no materialized .venv, so stub the + check (its own behaviour is covered by TestGates).""" + monkeypatch.setattr(runner, "_env_check", lambda root, job: None) - ``manifest.write_manifest`` reads several keys; we provide the ones - it touches without standing up a real container/decision pipeline. - The runner reads ``shell_command`` directly — substitution and - container wrapping happen at generation time. - """ + +def _cfg(project: Path, output_id: str = "foo", *, shell_command: str = "echo hi") -> dict: + """Minimal cfg matching what the Snakefile generator writes.""" return { "output_id": output_id, "output_type": "data", "universe_id": "u1", "recipe": "echo hi", "shell_command": shell_command, - "container_image": None, "decisions": {}, "code_version": "abc", + "env_version": load_environment(project).env_version, "git_sha": None, "lc_version": "test", } -def test_emit_lines_carry_sentinel(tmp_path: Path) -> None: - out_dir = tmp_path / "out" +def test_emit_lines_carry_sentinel(project: Path) -> None: + out_dir = project / "out" out_dir.mkdir() output, err = _capture( lambda: run_rule( @@ -63,7 +79,7 @@ def test_emit_lines_carry_sentinel(tmp_path: Path) -> None: universe="u1", output_dir=out_dir, inputs={}, - cfg=_cfg(shell_command="echo hello"), + cfg=_cfg(project, shell_command="echo hello"), ) ) assert err is None @@ -77,8 +93,8 @@ def test_emit_lines_carry_sentinel(tmp_path: Path) -> None: assert "✓ foo" in body -def test_failed_recipe_raises_and_emits_cross(tmp_path: Path) -> None: - out_dir = tmp_path / "out" +def test_failed_recipe_raises_and_emits_cross(project: Path) -> None: + out_dir = project / "out" out_dir.mkdir() output, err = _capture( lambda: run_rule( @@ -86,7 +102,7 @@ def test_failed_recipe_raises_and_emits_cross(tmp_path: Path) -> None: universe="u1", output_dir=out_dir, inputs={}, - cfg=_cfg(shell_command="false"), + cfg=_cfg(project, shell_command="false"), ) ) assert isinstance(err, subprocess.CalledProcessError) @@ -96,11 +112,11 @@ def test_failed_recipe_raises_and_emits_cross(tmp_path: Path) -> None: assert "exit=1" in body -def test_no_manifest_on_failure(tmp_path: Path) -> None: +def test_no_manifest_on_failure(project: Path) -> None: """A failing recipe must not leave a manifest behind — it would poison ``lc verify``'s chain check by claiming completion of an incomplete rule.""" - out_dir = tmp_path / "out" + out_dir = project / "out" out_dir.mkdir() _, err = _capture( lambda: run_rule( @@ -108,15 +124,15 @@ def test_no_manifest_on_failure(tmp_path: Path) -> None: universe="u1", output_dir=out_dir, inputs={}, - cfg=_cfg(shell_command="false"), + cfg=_cfg(project, shell_command="false"), ) ) assert err is not None assert not (out_dir / ".lightcone-manifest.json").exists() -def test_manifest_written_on_success(tmp_path: Path) -> None: - out_dir = tmp_path / "out" +def test_manifest_written_on_success(project: Path) -> None: + out_dir = project / "out" out_dir.mkdir() _, err = _capture( lambda: run_rule( @@ -124,15 +140,15 @@ def test_manifest_written_on_success(tmp_path: Path) -> None: universe="u1", output_dir=out_dir, inputs={}, - cfg=_cfg(shell_command=f"touch {out_dir}/data.txt"), + cfg=_cfg(project, shell_command=f"touch {out_dir}/data.txt"), ) ) assert err is None assert (out_dir / ".lightcone-manifest.json").is_file() -def test_recipe_stdout_and_stderr_both_forwarded(tmp_path: Path) -> None: - out_dir = tmp_path / "out" +def test_recipe_stdout_and_stderr_both_forwarded(project: Path) -> None: + out_dir = project / "out" out_dir.mkdir() output, err = _capture( lambda: run_rule( @@ -140,10 +156,195 @@ def test_recipe_stdout_and_stderr_both_forwarded(tmp_path: Path) -> None: universe="u1", output_dir=out_dir, inputs={}, - cfg=_cfg(shell_command="echo on-stdout; echo on-stderr 1>&2"), + cfg=_cfg(project, shell_command="echo on-stdout; echo on-stderr 1>&2"), + ) + ) + assert err is None + assert "on-stdout" in output + assert "on-stderr" in output + + +def test_manifest_records_hermeticity_and_attestation(project: Path) -> None: + """run_rule executes through the sandbox boundary and records what + actually ran; the worker-side runtime attestation is merged in.""" + from lightcone.engine.sandbox import _landlock + + if _landlock.abi() == 0: + pytest.skip("landlock unavailable on this kernel") + out_dir = project / "out" + out_dir.mkdir() + _, err = _capture( + lambda: run_rule( + rule_key="foo", + universe="u1", + output_dir=out_dir, + inputs={}, + cfg=_cfg(project, shell_command=f"touch {out_dir}/data.txt"), ) ) assert err is None - body = output - assert "on-stdout" in body - assert "on-stderr" in body + m = json.loads((out_dir / ".lightcone-manifest.json").read_text()) + h = m["hermeticity"] + assert h["mechanism"] == "landlock" + assert h["fs"] == "declared" + # Landlock cannot express a useful network deny — recorded, not + # pretended (spec's honest enum). + assert h["network"] == "unenforced" + assert h["landlock_abi"] >= 1 + assert h["exec_allowlist_version"] == 1 + assert m["platform"]["arch"] + assert m["python_build"].startswith("CPython") + assert m["worker_runtime"] == "host" + + +def test_no_sandbox_records_honestly( + project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(runner.NO_SANDBOX_ENV, "1") + out_dir = project / "out" + out_dir.mkdir() + _, err = _capture( + lambda: run_rule( + rule_key="foo", universe="u1", output_dir=out_dir, + inputs={}, + cfg=_cfg(project, shell_command=f"touch {out_dir}/data.txt"), + ) + ) + assert err is None + m = json.loads((out_dir / ".lightcone-manifest.json").read_text()) + assert m["hermeticity"]["mechanism"] == "none" + assert m["hermeticity"]["fs"] == "open" + + +# ---- the gates ------------------------------------------------------------- + + +class TestGates: + def test_pre_gate_aborts_on_env_drift(self, project: Path) -> None: + """A lock edited between generation and execution aborts before + the recipe runs.""" + out_dir = project / "out" + out_dir.mkdir() + cfg = _cfg(project, shell_command=f"touch {out_dir}/ran") + (project / "uv.lock").write_text( + (project / "uv.lock").read_text() + "# relock\n" + ) + output, err = _capture( + lambda: run_rule( + rule_key="foo", universe="u1", output_dir=out_dir, + inputs={}, cfg=cfg, + ) + ) + assert isinstance(err, RuleGateError) + assert "environment changed mid-run" in str(err) + assert not (out_dir / "ran").exists(), "recipe must not have run" + assert not (out_dir / ".lightcone-manifest.json").exists() + + def test_post_gate_blocks_manifest_on_mid_recipe_relock( + self, project: Path + ) -> None: + """A recipe (or concurrent edit) that changes the lock during + execution must not get a manifest — the double gate brackets the + recipe.""" + out_dir = project / "out" + out_dir.mkdir() + cfg = _cfg( + project, + shell_command=( + f"touch {out_dir}/data.txt && echo '# drift' >> uv.lock" + ), + ) + _, err = _capture( + lambda: run_rule( + rule_key="foo", universe="u1", output_dir=out_dir, + inputs={}, cfg=cfg, + ) + ) + assert isinstance(err, RuleGateError) + assert not (out_dir / ".lightcone-manifest.json").exists() + + def test_env_check_runs_uv_sync_check( + self, project: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Step 2 (direct): a true no-write env-vs-lock verification.""" + from lightcone.engine.job import RuleJob + + calls: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + calls.append(list(cmd)) + + class R: + returncode = 0 + stderr = "" + return R() + + (project / ".venv").mkdir() + monkeypatch.setattr(runner.subprocess, "run", fake_run) + job = RuleJob.from_cfg(_cfg(project)) + _real_env_check(project, job) + assert calls and calls[0][:5] == ["uv", "sync", "--locked", "--exact", "--check"] + + def test_env_check_fails_without_venv(self, project: Path) -> None: + from lightcone.engine.job import RuleJob + + job = RuleJob.from_cfg(_cfg(project)) + with pytest.raises(RuleGateError, match="never converged"): + _real_env_check(project, job) + + +# The autouse fixture stubs runner._env_check; keep a handle to the real +# implementation for the tests that exercise it directly. +_real_env_check = runner._env_check + + +# ---- offline overlay + sandbox flags --------------------------------------- + + +def test_recipe_env_has_offline_overlay_and_scrub( + project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Converge once, then never write: recipes see UV_OFFLINE=1 and + never the ambient UV_* steering surface.""" + monkeypatch.setenv("UV_INDEX_URL", "https://evil.example/simple") + out_dir = project / "out" + out_dir.mkdir() + output, err = _capture( + lambda: run_rule( + rule_key="foo", universe="u1", output_dir=out_dir, + inputs={}, + cfg=_cfg(project, shell_command="env | sort"), + ) + ) + assert err is None + assert "UV_OFFLINE=1" in output + assert "UV_PYTHON_DOWNLOADS=never" in output + assert "UV_INDEX_URL" not in output + + +def test_require_sandbox_refuses_before_exec( + project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Worker-side enforcement: when no mechanism is available, + --require-sandbox must refuse and the recipe must never run.""" + import lightcone.engine.sandbox.probe as probe_mod + from lightcone.engine.sandbox.model import SandboxCapability + + monkeypatch.setattr( + probe_mod, + "probe", + lambda: SandboxCapability(kind="none", detail="test"), + ) + monkeypatch.setenv(runner.REQUIRE_SANDBOX_ENV, "any") + out_dir = project / "out" + out_dir.mkdir() + _, err = _capture( + lambda: run_rule( + rule_key="foo", universe="u1", output_dir=out_dir, + inputs={}, + cfg=_cfg(project, shell_command=f"touch {out_dir}/ran"), + ) + ) + assert isinstance(err, RuleGateError) + assert "--require-sandbox" in str(err) + assert not (out_dir / "ran").exists() diff --git a/tests/test_sandbox_denial.py b/tests/test_sandbox_denial.py new file mode 100644 index 00000000..fd83366f --- /dev/null +++ b/tests/test_sandbox_denial.py @@ -0,0 +1,117 @@ +"""Pure-function tests for the denial UX renderer.""" +from __future__ import annotations + +from pathlib import Path + +from lightcone.engine.sandbox.denial import explain_failure, trailer +from lightcone.engine.sandbox.hints import HINT_TABLE_VERSION, HINTS, apt_hint +from lightcone.engine.sandbox.model import SandboxPolicy + + +def _policy(tmp_path: Path) -> SandboxPolicy: + project = tmp_path / "proj" + project.mkdir(exist_ok=True) + return SandboxPolicy( + read=(project,), + write=(tmp_path / "out",), + execute=(project / ".venv" / "bin",), + tmp_home=tmp_path / "home", + env={}, + fs_scope="declared", + exec_allowlist_version=1, + ) + + +class TestExplainFailure: + def test_tool_denial_renders_two_remedies(self, tmp_path: Path) -> None: + lines = explain_failure( + stdout="", + stderr="bash: line 1: /usr/bin/id: Permission denied", + policy=_policy(tmp_path), + ) + joined = "\n".join(lines) + assert "blocked by lc sandbox" in joined + assert "cannot execute /usr/bin/id" in joined + # Both remedies always shown; tool-first ordering here. + assert "[tool.lightcone.image]" in joined + assert "astra.yaml" in joined + assert joined.index("[tool.lightcone.image]") < joined.index("astra.yaml") + # Cost stated up front; escape hatches subdued at the end. + assert "podman required" in joined + assert "lc run --no-sandbox" in joined + + def test_known_tool_gets_apt_hint(self, tmp_path: Path) -> None: + # Rscript exists on this system? Use a synthetic bin-dir path that + # exists: fall back to a real allowlisted-tool-like case via + # /usr/bin/Rscript existence check. + target = Path("/usr/bin/Rscript") + if not target.exists(): + import pytest + + pytest.skip("Rscript not installed on this host") + lines = explain_failure( + stdout="", + stderr=f"bash: line 1: {target}: Permission denied", + policy=_policy(tmp_path), + ) + assert any("r-base-core" in line for line in lines) + + def test_data_denial_orders_input_remedy_first(self, tmp_path: Path) -> None: + data = tmp_path / "external" / "table.csv" + data.parent.mkdir() + data.write_text("x\n") + lines = explain_failure( + stdout=f"PermissionError: [Errno 13] Permission denied: '{data}'", + stderr="", + policy=_policy(tmp_path), + ) + joined = "\n".join(lines) + assert f"cannot read {data}" in joined + assert joined.index("astra.yaml") < joined.index("[tool.lightcone.image]") + + def test_in_policy_path_not_flagged(self, tmp_path: Path) -> None: + """A path inside the granted sets is an ordinary recipe error, + not a denial — no message (the trailer still fires elsewhere).""" + policy = _policy(tmp_path) + inside = policy.read[0] / "missing-but-in-project.txt" + lines = explain_failure( + stdout=f"FileNotFoundError: '{inside}'", + stderr="", + policy=policy, + ) + assert lines == [] + + def test_nonexistent_path_not_flagged(self, tmp_path: Path) -> None: + """Re-stat separates denial from typo: a path that doesn't exist + on the host is not a sandbox denial.""" + lines = explain_failure( + stdout="FileNotFoundError: '/no/such/path/anywhere'", + stderr="", + policy=_policy(tmp_path), + ) + assert lines == [] + + def test_command_not_found_resolved_via_which(self, tmp_path: Path) -> None: + """`foo: command not found` for a tool that exists outside the + sandbox PATH classifies as a tool denial.""" + lines = explain_failure( + stdout="", + stderr="bash: line 1: id: command not found", + policy=_policy(tmp_path), + ) + assert any("blocked by lc sandbox" in line for line in lines) + + +class TestTrailer: + def test_trailer_names_mechanism(self) -> None: + t = trailer("landlock") + assert "lc sandbox (landlock)" in t + assert "lc run --sandbox-debug" in t + + +class TestHints: + def test_capped_and_versioned(self) -> None: + assert HINT_TABLE_VERSION == 1 + assert len(HINTS) <= 20, "the hint table is capped, never open-ended" + assert apt_hint("latex") == "texlive-latex-base" + assert apt_hint("some-unknown-tool") is None diff --git a/tests/test_sandbox_enforcement.py b/tests/test_sandbox_enforcement.py new file mode 100644 index 00000000..1d3ffae5 --- /dev/null +++ b/tests/test_sandbox_enforcement.py @@ -0,0 +1,175 @@ +"""End-to-end Landlock enforcement through the real shim. + +These run unprivileged (Landlock needs no capabilities). Fixture +projects live under ``$HOME`` — NOT pytest's tmp_path — because the §7 +policy grants ``/tmp`` blanket-RW, which would mask project-tree +denials for a project living inside it. +""" +from __future__ import annotations + +import shutil +import sys +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from lightcone.engine.boundary import ExecScope +from lightcone.engine.sandbox import _landlock +from lightcone.engine.sandbox.exec_boundary import SandboxExecBoundary + +pytestmark = [ + pytest.mark.skipif(sys.platform != "linux", reason="Landlock is Linux-only"), + pytest.mark.skipif( + sys.platform == "linux" and _landlock.abi() == 0, + reason="Landlock unavailable on this kernel", + ), +] + + +@pytest.fixture +def home_project() -> Iterator[Path]: + root = Path(tempfile.mkdtemp(prefix="lc-sbx-", dir=Path.home())) + try: + project = root / "proj" + (project / "results" / "u1" / "foo").mkdir(parents=True) + (project / "data.txt").write_text("in-project data\n") + (root / "outside.txt").write_text("secret outside the project\n") + yield project + finally: + shutil.rmtree(root, ignore_errors=True) + + +def _run(project: Path, command: str, **scope_kwargs): # type: ignore[no-untyped-def] + scope_kwargs.setdefault("read_paths", ()) + scope = ExecScope( + project_root=project, + output_dir=project / "results" / "u1" / "foo", + **scope_kwargs, + ) + import os + + return SandboxExecBoundary().execute(command, scope, env=dict(os.environ)) + + +class TestFilesystem: + def test_write_in_own_output_allowed(self, home_project: Path) -> None: + r = _run(home_project, "echo done > results/u1/foo/out.txt") + assert r.returncode == 0, r.stderr + assert (home_project / "results" / "u1" / "foo" / "out.txt").exists() + + def test_project_tree_write_denied(self, home_project: Path) -> None: + """Sibling outputs, manifests, and astra.yaml are protected from + a misbehaving recipe.""" + r = _run(home_project, "echo overwrite > data.txt") + assert r.returncode != 0 + assert (home_project / "data.txt").read_text() == "in-project data\n" + + def test_project_read_allowed(self, home_project: Path) -> None: + r = _run(home_project, "cat data.txt") + assert r.returncode == 0 + assert "in-project data" in r.stdout + + def test_undeclared_read_denied(self, home_project: Path) -> None: + outside = home_project.parent / "outside.txt" + r = _run(home_project, f"cat {outside}") + assert r.returncode != 0 + assert "secret" not in r.stdout + + def test_declared_input_read_allowed(self, home_project: Path) -> None: + outside = home_project.parent / "outside.txt" + r = _run(home_project, f"cat {outside}", read_paths=(outside,)) + assert r.returncode == 0 + assert "secret" in r.stdout + + def test_writable_project_escalation(self, home_project: Path) -> None: + r = _run(home_project, "echo v2 > data.txt", writable_project=True) + assert r.returncode == 0, r.stderr + assert (home_project / "data.txt").read_text() == "v2\n" + + def test_real_home_not_readable(self, home_project: Path) -> None: + probe_file = Path.home() / ".lc-sandbox-test-canary" + probe_file.write_text("canary") + try: + r = _run(home_project, f"cat {probe_file}") + assert r.returncode != 0 + assert "canary" not in r.stdout + finally: + probe_file.unlink() + + +class TestExec: + def test_dynamically_linked_exec_succeeds(self, home_project: Path) -> None: + """/bin/ls is dynamically linked: this passing proves the ELF + loader tier — Landlock checks EXECUTE on the loader's open, so + without it every dynamic binary fails EACCES.""" + r = _run(home_project, "ls results") + assert r.returncode == 0, r.stderr + + def test_undeclared_tool_exec_denied(self, home_project: Path) -> None: + # /usr/bin/id exists and is readable (OS baseline) but is not in + # the exec allowlist. + assert Path("/usr/bin/id").exists() + r = _run(home_project, "/usr/bin/id") + assert r.returncode != 0 + assert "uid=" not in r.stdout + + def test_allowlisted_pipeline_works(self, home_project: Path) -> None: + r = _run( + home_project, + "printf 'b\\na\\n' | sort | head -1 > results/u1/foo/first.txt", + ) + assert r.returncode == 0, r.stderr + assert (home_project / "results/u1/foo/first.txt").read_text() == "a\n" + + +class TestPycache: + def test_no_pycache_in_tree_and_import_succeeds( + self, home_project: Path + ) -> None: + """PYTHONPYCACHEPREFIX (approved §7 amendment): in-tree imports + work at full speed with the tree read-only, and no __pycache__ + appears in the project.""" + (home_project / "mymod.py").write_text("VALUE = 41 + 1\n") + venv_bin = home_project / ".venv" / "bin" + venv_bin.mkdir(parents=True) + (venv_bin / "python").symlink_to(sys.executable) + r = _run( + home_project, + '.venv/bin/python -c "import mymod; print(mymod.VALUE)"', + ) + assert r.returncode == 0, r.stderr + assert "42" in r.stdout + assert not (home_project / "__pycache__").exists() + + +class TestAttestation: + def test_landlock_attested(self, home_project: Path) -> None: + r = _run(home_project, "true") + assert r.attestation.mechanism == "landlock" + assert r.attestation.fs == "declared" + assert r.attestation.network == "unenforced" + assert (r.attestation.landlock_abi or 0) >= 1 + + def test_denial_message_renders_for_tool(self, home_project: Path) -> None: + """The primary UI: a blocked exec explains itself with the + two-remedy message.""" + r = _run(home_project, "/usr/bin/id") + joined = "\n".join(r.notes) + assert "blocked by lc sandbox" in joined + assert "[tool.lightcone.image]" in joined + assert "astra.yaml" in joined + assert "lc run --sandbox-debug" in joined + + def test_trailer_fires_on_swallowed_error(self, home_project: Path) -> None: + """A recipe that swallows the PermissionError and exits nonzero + still gets the fixed trailer — a denial can never be invisible.""" + r = _run( + home_project, + "cat data.txt > /dev/null; echo overwrite > data.txt 2>/dev/null; exit 3", + ) + assert r.returncode == 3 + joined = "\n".join(r.notes) + assert "ran under the lc sandbox" in joined + assert "--sandbox-debug" in joined diff --git a/tests/test_sandbox_policy.py b/tests/test_sandbox_policy.py new file mode 100644 index 00000000..ceb2c323 --- /dev/null +++ b/tests/test_sandbox_policy.py @@ -0,0 +1,145 @@ +"""Pure-function tests for the sandbox policy builder.""" +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + +import pytest + +from lightcone.engine.boundary import ExecScope +from lightcone.engine.sandbox.policy import ( + EXEC_ALLOWLIST_V1, + EXEC_ALLOWLIST_VERSION, + build_policy, +) + + +@pytest.fixture +def scope(tmp_path: Path) -> ExecScope: + project = tmp_path / "proj" + (project / "results" / "u1" / "foo").mkdir(parents=True) + return ExecScope( + project_root=project, + output_dir=project / "results" / "u1" / "foo", + read_paths=(), + ) + + +def _cleanup(policy) -> None: # type: ignore[no-untyped-def] + shutil.rmtree(policy.tmp_home, ignore_errors=True) + + +def test_write_set(scope: ExecScope, tmp_path: Path) -> None: + policy = build_policy(scope, env_prefix=tmp_path / "noenv") + try: + writes = {str(p) for p in policy.write} + assert str(scope.output_dir.resolve()) in writes + assert "/tmp" in writes + assert str(policy.tmp_home) in writes + # The project tree is NOT writable by default. + assert str(scope.project_root.resolve()) not in writes + assert policy.fs_scope == "declared" + finally: + _cleanup(policy) + + +def test_writable_project_escalation(scope: ExecScope, tmp_path: Path) -> None: + escalated = ExecScope( + project_root=scope.project_root, + output_dir=scope.output_dir, + read_paths=(), + writable_project=True, + ) + policy = build_policy(escalated, env_prefix=tmp_path / "noenv") + try: + assert str(scope.project_root.resolve()) in {str(p) for p in policy.write} + assert policy.fs_scope == "project-rw" + finally: + _cleanup(policy) + + +def test_probe_scope_has_no_in_tree_write(tmp_path: Path) -> None: + """Probes (no output dir) write only to the tmp scope — never + in-tree.""" + project = tmp_path / "proj" + project.mkdir() + probe_scope = ExecScope(project_root=project, output_dir=None, read_paths=()) + policy = build_policy(probe_scope, env_prefix=tmp_path / "noenv") + try: + for p in policy.write: + assert not str(p).startswith(str(project.resolve())) + finally: + _cleanup(policy) + + +def test_home_xdg_contract(scope: ExecScope, tmp_path: Path) -> None: + """Fresh per-recipe HOME; matplotlib/astropy work on first import; + bytecode caches redirect to the tmp scope (the approved §7 + amendment); the real $HOME is simply not granted.""" + policy = build_policy(scope, env_prefix=tmp_path / "noenv") + try: + home = Path(policy.env["HOME"]) + assert home == policy.tmp_home + assert home.is_dir() + for key in ( + "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME", + "MPLCONFIGDIR", "PYTHONPYCACHEPREFIX", + ): + assert Path(policy.env[key]).is_dir() + assert str(policy.env[key]).startswith(str(home)) + assert str(Path.home()) not in {str(p) for p in policy.read} + assert str(Path.home()) not in {str(p) for p in policy.write} + finally: + _cleanup(policy) + + +def test_exec_allowlist_versioned(scope: ExecScope, tmp_path: Path) -> None: + policy = build_policy(scope, env_prefix=tmp_path / "noenv") + try: + assert policy.exec_allowlist_version == EXEC_ALLOWLIST_VERSION == 1 + assert "bash" in EXEC_ALLOWLIST_V1 and "awk" in EXEC_ALLOWLIST_V1 + finally: + _cleanup(policy) + + +@pytest.mark.skipif(sys.platform != "linux", reason="ELF loaders are Linux") +def test_elf_loader_tier_present(scope: ExecScope, tmp_path: Path) -> None: + """Without the loader every dynamically linked binary fails EACCES — + the loader tier must be in the exec set.""" + policy = build_policy(scope, env_prefix=tmp_path / "noenv") + try: + assert any("ld-linux" in str(p) or "ld-musl" in str(p) for p in policy.execute) + finally: + _cleanup(policy) + + +def test_venv_bin_and_real_interpreter_granted(tmp_path: Path) -> None: + """The env bin dir gets a directory grant, and the symlink-resolved + interpreter install root is granted too (realpath every policy + path).""" + project = tmp_path / "proj" + (project / "out").mkdir(parents=True) + venv_bin = project / ".venv" / "bin" + venv_bin.mkdir(parents=True) + (venv_bin / "python").symlink_to(sys.executable) + scope = ExecScope( + project_root=project, output_dir=project / "out", read_paths=() + ) + policy = build_policy(scope, env_prefix=project / ".venv") + try: + execs = {str(p) for p in policy.execute} + assert str(venv_bin.resolve()) in execs + real_root = Path(sys.executable).resolve().parent.parent + assert str(real_root) in execs + finally: + _cleanup(policy) + + +def test_all_paths_realpathed(scope: ExecScope, tmp_path: Path) -> None: + policy = build_policy(scope, env_prefix=tmp_path / "noenv") + try: + for p in (*policy.read, *policy.write, *policy.execute): + assert p == p.resolve(), p + finally: + _cleanup(policy) diff --git a/tests/test_sandbox_shim.py b/tests/test_sandbox_shim.py new file mode 100644 index 00000000..f97e1825 --- /dev/null +++ b/tests/test_sandbox_shim.py @@ -0,0 +1,75 @@ +"""Tests for the exec shim (`python -m lightcone._sandbox_exec`).""" +from __future__ import annotations + +import os +import subprocess +import sys + +SHIM = [sys.executable, "-m", "lightcone._sandbox_exec"] + + +def _run_shim(*argv: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [*SHIM, "--", *argv], + capture_output=True, + text=True, + env={**os.environ, **env}, + ) + + +def test_mode_none_passthrough() -> None: + """LC_SANDBOX_MODE=none keeps the argv shape uniform: straight exec.""" + r = _run_shim("echo", "shim-ok", env={"LC_SANDBOX_MODE": "none"}) + assert r.returncode == 0 + assert r.stdout.strip() == "shim-ok" + + +def test_missing_fd_exits_97() -> None: + """The never-silent guard: a landlock exec whose ruleset FD did not + survive must fail with the reserved setup exit code — never proceed + unsandboxed.""" + r = _run_shim("true", env={"LC_SANDBOX_MODE": "landlock"}) + assert r.returncode == 97 + assert "lc sandbox setup failed" in r.stderr + + +def test_bad_fd_exits_97() -> None: + r = _run_shim( + "true", env={"LC_SANDBOX_MODE": "landlock", "LC_SANDBOX_FD": "999"} + ) + assert r.returncode == 97 + assert "did not survive" in r.stderr + + +def test_unknown_mode_exits_97() -> None: + r = _run_shim("true", env={"LC_SANDBOX_MODE": "bogus"}) + assert r.returncode == 97 + + +def test_no_command_exits_97() -> None: + r = subprocess.run( + [*SHIM, "--"], + capture_output=True, + text=True, + env={**os.environ, "LC_SANDBOX_MODE": "none"}, + ) + assert r.returncode == 97 + + +def test_shim_constants_match_bindings() -> None: + """The shim duplicates the restrict-side constants on purpose (no + engine imports inside the exec path) — parity is pinned here.""" + from lightcone import _sandbox_exec + from lightcone.engine.sandbox import _landlock + + assert _sandbox_exec._SYS_LANDLOCK_RESTRICT_SELF == ( + _landlock.SYS_LANDLOCK_RESTRICT_SELF + ) + assert _sandbox_exec._PR_SET_NO_NEW_PRIVS == 38 + + +def test_shim_scrubs_control_env() -> None: + """The LC_SANDBOX_* control vars must not leak into the recipe.""" + r = _run_shim("env", env={"LC_SANDBOX_MODE": "none"}) + assert r.returncode == 0 + assert "LC_SANDBOX_MODE" not in r.stdout diff --git a/tests/test_scratch.py b/tests/test_scratch.py index 3a884fb3..2dcc24d3 100644 --- a/tests/test_scratch.py +++ b/tests/test_scratch.py @@ -82,30 +82,6 @@ def test_project_config_expands( assert resolve_scratch_root(project) == tmp_path / "expanded" -def test_site_default_resolves_when_env_set( - monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path -) -> None: - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - monkeypatch.setenv("SCRATCH", str(tmp_path / "lustre")) - import socket - monkeypatch.setattr(socket, "gethostname", lambda: "perlmutter-login01") - assert resolve_scratch_root(project) == tmp_path / "lustre" - - -def test_site_default_falls_through_when_env_missing( - monkeypatch: pytest.MonkeyPatch, project: Path -) -> None: - """If a known site's scratch_root is ``$SCRATCH`` and ``SCRATCH`` is - not set, the unexpanded ``$SCRATCH`` mustn't become a literal path — - we fall through to the tempdir fallback instead. - """ - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - monkeypatch.delenv("SCRATCH", raising=False) - import socket - monkeypatch.setattr(socket, "gethostname", lambda: "perlmutter-login01") - resolved = resolve_scratch_root(project) - assert "$" not in str(resolved) - assert resolved == Path(tempfile.gettempdir()) def test_fallback_to_tempdir(monkeypatch: pytest.MonkeyPatch, project: Path) -> None: diff --git a/tests/test_seatbelt.py b/tests/test_seatbelt.py new file mode 100644 index 00000000..f90390a5 --- /dev/null +++ b/tests/test_seatbelt.py @@ -0,0 +1,105 @@ +"""Seatbelt profile generation (runs on Linux; enforcement smoke lives +in macOS CI).""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from lightcone.engine.boundary import ExecScope +from lightcone.engine.sandbox.policy import build_policy +from lightcone.engine.sandbox.seatbelt import generate_profile + + +@pytest.fixture +def profile(tmp_path: Path) -> str: + project = tmp_path / "proj" + (project / "results" / "u1" / "foo").mkdir(parents=True) + scope = ExecScope( + project_root=project, + output_dir=project / "results" / "u1" / "foo", + read_paths=(), + ) + policy = build_policy(scope, env_prefix=project / ".venv") + try: + return generate_profile(policy) + finally: + import shutil + + shutil.rmtree(policy.tmp_home, ignore_errors=True) + + +class TestProfileShape: + def test_deny_default(self, profile: str) -> None: + assert "(version 1)" in profile + assert "(deny default)" in profile + + def test_loopback_only_network(self, profile: str) -> None: + """'denied' means non-loopback blocked, loopback intact — the + spec's meaning; in-recipe LocalCluster keeps working.""" + assert '(allow network-outbound (remote ip "localhost:*"))' in profile + assert '(deny network-outbound (remote ip "*:*"))' in profile + assert "network-bind" in profile + + def test_ipc_for_multiprocessing(self, profile: str) -> None: + assert "(allow ipc-posix-shm*)" in profile + assert "(allow ipc-posix-sem*)" in profile + + def test_project_readable_output_writable( + self, profile: str, tmp_path: Path + ) -> None: + assert str(tmp_path / "proj") in profile + assert "file-write*" in profile + + def test_dyld_executable(self, profile: str) -> None: + assert '"/usr/lib/dyld"' in profile + + +@pytest.mark.darwin +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS enforcement smoke") +class TestEnforcementSmoke: + """Runs in the macOS CI workflow only.""" + + def test_write_outside_denied(self, tmp_path: Path) -> None: + import os + + from lightcone.engine.sandbox.exec_boundary import SandboxExecBoundary + + project = tmp_path / "proj" + (project / "results" / "u1" / "foo").mkdir(parents=True) + (project / "data.txt").write_text("v1\n") + scope = ExecScope( + project_root=project, + output_dir=project / "results" / "u1" / "foo", + read_paths=(), + ) + boundary = SandboxExecBoundary() + ok = boundary.execute( + "echo hi > results/u1/foo/x.txt", scope, env=dict(os.environ) + ) + assert ok.returncode == 0, ok.stderr + denied = boundary.execute( + "echo overwrite > data.txt", scope, env=dict(os.environ) + ) + assert denied.returncode != 0 + assert (project / "data.txt").read_text() == "v1\n" + + def test_non_loopback_network_denied(self, tmp_path: Path) -> None: + import os + + from lightcone.engine.sandbox.exec_boundary import SandboxExecBoundary + + project = tmp_path / "proj" + (project / "results" / "u1" / "foo").mkdir(parents=True) + scope = ExecScope( + project_root=project, + output_dir=project / "results" / "u1" / "foo", + read_paths=(), + ) + r = SandboxExecBoundary().execute( + "curl --max-time 3 -sS https://1.1.1.1 && echo REACHED", + scope, + env=dict(os.environ), + ) + assert "REACHED" not in r.stdout diff --git a/tests/test_site_registry.py b/tests/test_site_registry.py deleted file mode 100644 index 939dbeeb..00000000 --- a/tests/test_site_registry.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Tests for the site registry — site detection and the HostSite wrapper.""" -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path - -import pytest - -from lightcone.engine import site_registry -from lightcone.engine.site_registry import ( - HostSite, - detect_current_site, - detect_site, -) - - -@pytest.fixture -def fake_hostname(monkeypatch: pytest.MonkeyPatch) -> Callable[[str], None]: - """Return a setter that pins ``socket.gethostname`` for the test.""" - - def _set(name: str) -> None: - monkeypatch.setattr(site_registry.socket, "gethostname", lambda: name) - - return _set - - -class TestDetectSite: - def test_matches_perlmutter_substring(self) -> None: - assert detect_site("login29.chn.perlmutter.nersc.gov") == "perlmutter" - - def test_matches_saul_pattern(self) -> None: - assert detect_site("saul01") == "perlmutter" - - def test_unknown_host(self) -> None: - assert detect_site("generic-laptop") is None - - def test_local_site_skipped(self) -> None: - # "local" has backend=local and is excluded from auto-detection. - assert detect_site("local") is None - - -class TestHostSite: - def test_matched_site_is_truthy(self) -> None: - site = HostSite(key="perlmutter", defaults={"display_name": "NERSC Perlmutter"}) - assert bool(site) is True - - def test_unmatched_site_is_falsy(self) -> None: - assert bool(HostSite(key=None)) is False - - def test_get_returns_field(self) -> None: - site = HostSite(key="perlmutter", defaults={"container_runtime": "podman-hpc"}) - assert site.get("container_runtime") == "podman-hpc" - - def test_get_missing_field_returns_default(self) -> None: - site = HostSite(key="perlmutter", defaults={}) - assert site.get("missing", "fallback") == "fallback" - assert site.get("missing") is None - - def test_display_name_from_defaults(self) -> None: - site = HostSite(key="perlmutter", defaults={"display_name": "NERSC Perlmutter"}) - assert site.display_name == "NERSC Perlmutter" - - def test_display_name_falls_back_to_key(self) -> None: - site = HostSite(key="perlmutter", defaults={}) - assert site.display_name == "perlmutter" - - def test_display_name_for_unknown_site(self) -> None: - assert HostSite(key=None).display_name == "unknown" - - -class TestDetectCurrentSite: - def test_known_host_returns_populated_site( - self, fake_hostname: Callable[[str], None] - ) -> None: - fake_hostname("login29.chn.perlmutter.nersc.gov") - site = detect_current_site() - assert site - assert site.key == "perlmutter" - assert site.get("container_runtime") == "podman-hpc" - assert site.display_name == "NERSC Perlmutter" - - def test_unknown_host_returns_empty_site( - self, fake_hostname: Callable[[str], None] - ) -> None: - fake_hostname("generic-laptop") - site = detect_current_site() - assert not site - assert site.key is None - assert site.get("container_runtime") is None - - def test_unknown_host_get_returns_default( - self, fake_hostname: Callable[[str], None] - ) -> None: - # Field access on an unmatched site shouldn't require an explicit - # truthiness guard at every call site — that's the whole point of - # returning an empty HostSite rather than None. - fake_hostname("generic-laptop") - assert detect_current_site().get("scratch_root", "/tmp") == "/tmp" - - -# ---- env-marker detection (JupyterHub deployments) ------------------------ - - -def test_detect_site_from_env_matches_jupyterhub( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from lightcone.engine.site_registry import detect_site_from_env - - monkeypatch.delenv("DASK_GATEWAY__ADDRESS", raising=False) - assert detect_site_from_env() is None - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - assert detect_site_from_env() == "jupyterhub" - - -def test_env_markers_win_over_hostname(monkeypatch: pytest.MonkeyPatch) -> None: - """A pod's hostname is noise; the injected env is the signal.""" - from lightcone.engine.site_registry import detect_current_site - - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setattr( - "lightcone.engine.site_registry.socket.gethostname", - lambda: "login29.chn.perlmutter.nersc.gov", - ) - site = detect_current_site() - assert site.key == "jupyterhub" - assert site.get("container_runtime") == "kubernetes" - assert site.get("scratch_root") == "$HOME" - - -def test_no_markers_falls_back_to_hostname( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from lightcone.engine.site_registry import detect_current_site - - monkeypatch.delenv("DASK_GATEWAY__ADDRESS", raising=False) - monkeypatch.setattr( - "lightcone.engine.site_registry.socket.gethostname", - lambda: "login29.chn.perlmutter.nersc.gov", - ) - assert detect_current_site().key == "perlmutter" - - -def test_hub_scratch_resolves_to_home( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """No separate scratch space on the hub — home IS the shared volume.""" - import os - - from lightcone.engine.scratch import resolve_scratch_root - - project = tmp_path / "proj" - project.mkdir() - monkeypatch.delenv("LIGHTCONE_SCRATCH", raising=False) - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - assert resolve_scratch_root(project) == Path(os.environ["HOME"]) diff --git a/tests/test_snakefile.py b/tests/test_snakefile.py index aa64128c..71ed2bce 100644 --- a/tests/test_snakefile.py +++ b/tests/test_snakefile.py @@ -7,6 +7,7 @@ import pytest import yaml +from conftest import PYPROJECT_MIN, PYTHON_VERSION_MIN, UV_LOCK_MIN from lightcone.engine.snakefile import generate, render_recipe @@ -14,6 +15,12 @@ def _spec(project_root: Path, spec: dict[str, Any]) -> None: project_root.mkdir(parents=True, exist_ok=True) (project_root / "astra.yaml").write_text(yaml.safe_dump(spec)) + # The generator loads the project environment: every fixture project + # carries the minimal uv scaffold. + if not (project_root / "pyproject.toml").exists(): + (project_root / "pyproject.toml").write_text(PYPROJECT_MIN) + (project_root / "uv.lock").write_text(UV_LOCK_MIN) + (project_root / ".python-version").write_text(PYTHON_VERSION_MIN) def test_generate_simple_spec(tmp_path: Path) -> None: @@ -97,73 +104,44 @@ def test_generate_includes_recipe_in_cfg(tmp_path: Path) -> None: assert f"lc_code_version={cfg['foo']['u1']['code_version']}" in sh -def test_generate_no_container_directive_emitted(tmp_path: Path) -> None: - """We own container invocation; the Snakemake ``container:`` directive - must never be emitted (we don't use --sdm apptainer).""" +def test_generate_ignores_legacy_container_keys(tmp_path: Path) -> None: + """ASTRA carries no container information — legacy ``container:`` + keys (top-level, per-recipe) are ignored; the environment lives in + pyproject.toml. The Snakemake ``container:`` directive is never + emitted either.""" _spec( tmp_path, { + "container": "Containerfile.legacy", "outputs": [ { "id": "foo", "recipe": {"command": "echo", "container": "python:3.12-slim"}, } - ] + ], }, ) - snakefile_path, _ = generate(tmp_path, universes=["u1"], runtime="podman") + snakefile_path, cfg_path = generate(tmp_path, universes=["u1"]) text = snakefile_path.read_text() assert "container:" not in text - - -def test_generate_wraps_recipe_with_runtime(tmp_path: Path) -> None: - """When a runtime is configured and the recipe has a container, the - wrapped shell command in cfg invokes the runtime with the image — - and the v0.0.7 ``{output}`` placeholder has been substituted to a - concrete per-universe path before the wrap.""" - _spec( - tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": { - "command": "echo hi > {output}/data.txt", - "container": "python:3.12-slim", - }, - } - ] - }, - ) - _, cfg_path = generate(tmp_path, universes=["u1"], runtime="podman") cfg = json.loads(cfg_path.read_text()) - sh = cfg["foo"]["u1"]["shell_command"] - assert "podman run --rm" in sh - assert "python:3.12-slim" in sh - # ``{output}`` is rendered at gen time to the concrete per-universe - # path; no placeholder survives the wrap. - assert "results/u1/foo/data.txt" in sh - assert "{output}" not in sh - # The code_version breadcrumb is prefixed onto the wrapped command. - assert f"lc_code_version={cfg['foo']['u1']['code_version']}" in sh + assert "container_image" not in cfg["foo"]["u1"] + assert "python:3.12-slim" not in cfg["foo"]["u1"]["shell_command"] -def test_generate_no_wrap_for_runtime_none(tmp_path: Path) -> None: +def test_generate_never_wraps_recipes(tmp_path: Path) -> None: + """Recipes are bare at generation time — enforcement is applied at + exec time by the boundary; in containerized mode the whole stack + already runs inside the image.""" _spec( tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": {"command": "echo hi", "container": "python:3.12-slim"}, - } - ] - }, + {"outputs": [{"id": "foo", "recipe": {"command": "echo hi"}}]}, ) - _, cfg_path = generate(tmp_path, universes=["u1"], runtime="none") + _, cfg_path = generate(tmp_path, universes=["u1"]) cfg = json.loads(cfg_path.read_text()) sh = cfg["foo"]["u1"]["shell_command"] assert sh.endswith("echo hi") + assert "podman" not in sh assert f"lc_code_version={cfg['foo']['u1']['code_version']}" in sh @@ -249,34 +227,73 @@ def test_recipe_edit_changes_params_for_rerun_trigger(tmp_path: Path) -> None: assert cfg_v1["shell_command"] != cfg_v2["shell_command"] -def test_containerfile_edit_changes_code_version(tmp_path: Path) -> None: - """Editing a Containerfile changes ``code_version`` so ``lc status`` - reports stale and the manifest records the image content faithfully. - """ - containerfile = tmp_path / "Containerfile" - containerfile.write_text("FROM python:3.12-slim\n") - _spec( - tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": {"command": "echo", "container": "Containerfile"}, - } - ] - }, - ) - _, cfg_path_v1 = generate(tmp_path, universes=["u1"], runtime="podman") +def test_environment_edit_changes_code_version(tmp_path: Path) -> None: + """An environment edit (env_version moves) changes every output's + ``code_version`` so ``lc status`` reports stale — the environment is + inside the identity.""" + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) + _, cfg_path_v1 = generate(tmp_path, universes=["u1"]) cv_v1 = json.loads(cfg_path_v1.read_text())["foo"]["u1"]["code_version"] - containerfile.write_text("FROM python:3.12-slim\nRUN pip install numpy\n") - _, cfg_path_v2 = generate(tmp_path, universes=["u1"], runtime="podman") + (tmp_path / "uv.lock").write_text( + (tmp_path / "uv.lock").read_text() + "\n# dependency drift\n" + ) + _, cfg_path_v2 = generate(tmp_path, universes=["u1"]) cv_v2 = json.loads(cfg_path_v2.read_text())["foo"]["u1"]["code_version"] - assert cv_v1 != cv_v2, ( - "code_version must change when the Containerfile contents change " - "so that lc status correctly reports stale." + assert cv_v1 != cv_v2 + + +def test_cfg_carries_env_identity(tmp_path: Path) -> None: + """Every rule's cfg carries env_version (the mid-run gates' baseline) + plus the provenance capture fields.""" + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) + _, cfg_path = generate(tmp_path, universes=["u1"]) + entry = json.loads(cfg_path.read_text())["foo"]["u1"] + assert entry["env_version"].startswith("sha256:") + assert entry["writable_project"] is False + assert "git_dirty" in entry + assert entry["sdist_built"] == [] + + +def test_writable_project_flows_into_cfg_and_cv(tmp_path: Path) -> None: + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) + _, cfg_v1 = generate(tmp_path, universes=["u1"]) + cv_plain = json.loads(cfg_v1.read_text())["foo"]["u1"]["code_version"] + + (tmp_path / "pyproject.toml").write_text( + (tmp_path / "pyproject.toml").read_text() + + '\n[tool.lightcone.sandbox]\nwritable-project = ["foo"]\n' ) + _, cfg_v2 = generate(tmp_path, universes=["u1"]) + entry = json.loads(cfg_v2.read_text())["foo"]["u1"] + assert entry["writable_project"] is True + assert entry["code_version"] != cv_plain + + +def test_writable_project_unknown_output_refused(tmp_path: Path) -> None: + from lightcone.engine.environment import ProjectEnvironmentError + + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) + (tmp_path / "pyproject.toml").write_text( + (tmp_path / "pyproject.toml").read_text() + + '\n[tool.lightcone.sandbox]\nwritable-project = ["nope"]\n' + ) + with pytest.raises(ProjectEnvironmentError, match="undeclared"): + generate(tmp_path, universes=["u1"]) + + +def test_lock_refusal_blocks_generation(tmp_path: Path) -> None: + from lightcone.engine.environment import ProjectEnvironmentError + + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) + (tmp_path / "uv.lock").write_text( + (tmp_path / "uv.lock").read_text() + + '\n[[package]]\nname = "local-dep"\nversion = "0"\n' + + 'source = { directory = "../local-dep" }\n' + ) + with pytest.raises(ProjectEnvironmentError, match="unauditable"): + generate(tmp_path, universes=["u1"]) def test_validation_runs_via_run_rule(tmp_path: Path) -> None: @@ -563,37 +580,4 @@ def test_returns_none_when_empty_url(self, git_remote) -> None: assert git_remote("\n") is None -def test_generate_kubernetes_runtime_unwrapped_with_registry_ref( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """On the kubernetes runtime the worker pod runs the project image: - recipes stay unwrapped, and the image identity that flows into - code_version is the registry ref, so a Containerfile edit (new ref) - still triggers reruns.""" - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - _spec( - tmp_path, - { - "name": "proj", - "container": "Containerfile", - "outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}], - }, - ) - monkeypatch.setenv( - "LIGHTCONE_REGISTRY", "europe-west1-docker.pkg.dev/hub/images" - ) - _, cfg_path = generate(tmp_path, universes=["u1"], runtime="kubernetes") - entry = json.loads(cfg_path.read_text())["foo"]["u1"] - assert "docker run" not in entry["shell_command"] - assert "podman" not in entry["shell_command"] - assert "echo foo" in entry["shell_command"] - # The declared spec is what the manifest records — unchanged. - assert entry["container_image"] == "Containerfile" - - # Same spec, docker runtime (local tag, no registry): different - # image identity → different code_version. - monkeypatch.delenv("LIGHTCONE_REGISTRY") - _, cfg_path2 = generate(tmp_path, universes=["u1"], runtime="docker") - entry2 = json.loads(cfg_path2.read_text())["foo"]["u1"] - assert entry["code_version"] != entry2["code_version"] diff --git a/tests/test_status.py b/tests/test_status.py index afcd3078..5da12ac8 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -4,9 +4,10 @@ from pathlib import Path from typing import Any -import pytest import yaml +from conftest import PYPROJECT_MIN, PYTHON_VERSION_MIN, UV_LOCK_MIN +from lightcone.engine.environment import load_environment from lightcone.engine.manifest import code_version, write_manifest from lightcone.engine.status import OutputStatus, get_output_status @@ -14,6 +15,10 @@ def _write_spec(project_root: Path, spec: dict[str, Any]) -> None: project_root.mkdir(parents=True, exist_ok=True) (project_root / "astra.yaml").write_text(yaml.safe_dump(spec)) + if not (project_root / "pyproject.toml").exists(): + (project_root / "pyproject.toml").write_text(PYPROJECT_MIN) + (project_root / "uv.lock").write_text(UV_LOCK_MIN) + (project_root / ".python-version").write_text(PYTHON_VERSION_MIN) def _materialize( @@ -23,15 +28,15 @@ def _materialize( *, recipe: str, decisions: dict[str, Any] | None = None, - container_image: str | None = None, ) -> Path: out = project_root / "results" / universe_id / output_id out.mkdir(parents=True, exist_ok=True) (out / "data.txt").write_text("output bytes") + env_version = load_environment(project_root).env_version cv = code_version( recipe=recipe, - container_image=container_image, decisions=decisions or {}, + env_version=env_version, ) write_manifest( output_dir=out, @@ -40,9 +45,9 @@ def _materialize( "output_id": output_id, "universe_id": universe_id, "recipe": recipe, - "container_image": container_image, "decisions": decisions or {}, "code_version": cv, + "env_version": env_version, "git_sha": "abc", "lc_version": "0.0", }, @@ -166,38 +171,38 @@ def test_status_universe_specific(tmp_path: Path) -> None: assert statuses[0].status == "missing" -def test_status_ok_on_kubernetes_deployment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """On a hub, `lc run` hashes the registry ref into code_version; - status must resolve the image identity the same way or every - freshly materialized output reads as stale (live-hub regression).""" - from lightcone.engine.container import registry_image_ref - monkeypatch.setattr(Path, "home", lambda: tmp_path / "home") - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setenv( - "LIGHTCONE_REGISTRY", "europe-west1-docker.pkg.dev/hub/images" - ) + + +def test_status_pre_migration_manifest(tmp_path: Path) -> None: + """An earlier-schema manifest surfaces distinctly — not ok, not a + bare stale.""" + import json _write_spec( - tmp_path, - { - "name": "proj", - "container": "Containerfile", - "outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}], - }, + tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]} ) - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - ref = registry_image_ref( - "proj", - tmp_path / "Containerfile", - tmp_path, - registry="europe-west1-docker.pkg.dev/hub/images", + out = tmp_path / "results" / "u1" / "foo" + out.mkdir(parents=True) + (out / "data.txt").write_text("x") + (out / ".lightcone-manifest.json").write_text( + json.dumps({"schema_version": 1, "code_version": "sha256:old"}) ) - _materialize( - tmp_path, "foo", "baseline", recipe="echo foo", container_image=ref + statuses = list(get_output_status(tmp_path, universe_id="u1")) + assert statuses[0].status == "pre_migration" + + +def test_env_blast_radius_counts_env_drift(tmp_path: Path) -> None: + from lightcone.engine.status import env_blast_radius + + _write_spec( + tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]} ) + _materialize(tmp_path, "foo", "u1", recipe="echo") + assert env_blast_radius(tmp_path, universes=["u1"]) == 0 - statuses = list(get_output_status(tmp_path, universe_id="baseline")) - assert [s.status for s in statuses] == ["ok"] + # An environment edit stales every materialized output. + (tmp_path / "uv.lock").write_text( + (tmp_path / "uv.lock").read_text() + "# drift\n" + ) + assert env_blast_radius(tmp_path, universes=["u1"]) == 1 diff --git a/tests/test_verify.py b/tests/test_verify.py index 37ef6dca..9d3bce97 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -13,6 +13,8 @@ ) from lightcone.engine.verify import VerifyResult, verify_outputs +_ENV = "sha256:" + "ee" * 32 + def _spec(project_root: Path, spec: dict[str, Any]) -> None: project_root.mkdir(parents=True, exist_ok=True) @@ -37,11 +39,11 @@ def _materialize( "output_id": output_id, "universe_id": universe_id, "recipe": recipe, - "container_image": None, "decisions": {}, "code_version": code_version( - recipe=recipe, container_image=None, decisions={} + recipe=recipe, decisions={}, env_version=_ENV ), + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -112,11 +114,11 @@ def test_verify_detects_broken_chain(tmp_path: Path) -> None: "output_id": "upstream", "universe_id": "u1", "recipe": "echo u", - "container_image": None, "decisions": {}, "code_version": code_version( - recipe="echo u", container_image=None, decisions={} + recipe="echo u", decisions={}, env_version=_ENV ), + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -212,11 +214,11 @@ def test_verify_detects_broken_chain_for_qualified_input(tmp_path: Path) -> None "output_id": "real", "universe_id": "u1", "recipe": "echo r", - "container_image": None, "decisions": {}, "code_version": code_version( - recipe="echo r", container_image=None, decisions={} + recipe="echo r", decisions={}, env_version=_ENV ), + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -236,11 +238,11 @@ def test_verify_detects_broken_chain_for_qualified_input(tmp_path: Path) -> None "output_id": "real", "universe_id": "u1", "recipe": "echo r", - "container_image": None, "decisions": {}, "code_version": code_version( - recipe="echo r", container_image=None, decisions={} + recipe="echo r", decisions={}, env_version=_ENV ), + "env_version": _ENV, "git_sha": "g", "lc_version": "0.0", }, @@ -294,3 +296,80 @@ def test_verifyresult_dataclass(tmp_path: Path) -> None: assert r.output_id == "foo" assert r.passed assert r.failure is None + + +def test_verify_notes_pre_migration_still_checks_hashes(tmp_path: Path) -> None: + """A v1-era manifest still carries data_version — verify checks it + and reports the pre_migration note rather than failing outright.""" + import json + + from lightcone.engine.manifest import sha256_dir + + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) + out = tmp_path / "results" / "u1" / "foo" + out.mkdir(parents=True) + (out / "data.txt").write_text("x") + (out / MANIFEST_FILENAME).write_text( + json.dumps( + { + "schema_version": 1, + "code_version": "sha256:old", + "data_version": sha256_dir(out), + "input_versions": {}, + } + ) + ) + results = list(verify_outputs(tmp_path, universe_id="u1")) + assert results[0].passed + assert "pre_migration" in results[0].notes + assert "unsandboxed" in results[0].notes + + +def test_verify_notes_dirty_tree_and_unsandboxed(tmp_path: Path) -> None: + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo hi"}}]}) + out = tmp_path / "results" / "u1" / "foo" + out.mkdir(parents=True, exist_ok=True) + (out / "data.txt").write_text("x") + write_manifest( + output_dir=out, + inputs={}, + cfg={ + "output_id": "foo", + "universe_id": "u1", + "recipe": "echo hi", + "decisions": {}, + "code_version": "sha256:c", + "env_version": _ENV, + "git_dirty": True, + }, + ) + results = list(verify_outputs(tmp_path, universe_id="u1")) + assert results[0].passed + assert "dirty_tree" in results[0].notes + assert "unsandboxed" in results[0].notes + + +def test_verify_no_notes_when_sandboxed_and_clean(tmp_path: Path) -> None: + _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo hi"}}]}) + out = tmp_path / "results" / "u1" / "foo" + out.mkdir(parents=True, exist_ok=True) + (out / "data.txt").write_text("x") + write_manifest( + output_dir=out, + inputs={}, + cfg={ + "output_id": "foo", + "universe_id": "u1", + "recipe": "echo hi", + "decisions": {}, + "code_version": "sha256:c", + "env_version": _ENV, + "git_dirty": False, + }, + hermeticity={ + "mechanism": "landlock", "fs": "declared", "network": "unenforced", + }, + ) + results = list(verify_outputs(tmp_path, universe_id="u1")) + assert results[0].passed + assert results[0].notes == () diff --git a/tests/test_wrroc.py b/tests/test_wrroc.py index 733fadda..999e9551 100644 --- a/tests/test_wrroc.py +++ b/tests/test_wrroc.py @@ -17,6 +17,8 @@ export_wrroc, ) +_ENV = "sha256:" + "ee" * 32 + # --------------------------------------------------------------------------- # Fixtures: tiny project + materialized outputs # --------------------------------------------------------------------------- @@ -51,8 +53,8 @@ def _materialize( (out / "data.txt").write_text(body) cv = code_version( recipe=recipe, - container_image=container_image, decisions=decisions or {}, + env_version=_ENV, ) write_manifest( output_dir=out, @@ -61,9 +63,10 @@ def _materialize( "output_id": output_id, "universe_id": universe_id, "recipe": recipe, - "container_image": container_image, + "image_tag": container_image, "decisions": decisions or {}, "code_version": cv, + "env_version": _ENV, "git_sha": "abc1234", "lc_version": "0.0.1", }, @@ -424,13 +427,13 @@ def subanalysis_project(self, tmp_path: Path) -> Path: sub_out_dir = sub_dir / "results" / "baseline" / "sub_out" sub_out_dir.mkdir(parents=True) (sub_out_dir / "data.txt").write_text("sub bytes") - cv = code_version(recipe="echo s", container_image=None, decisions={}) + cv = code_version(recipe="echo s", decisions={}, env_version=_ENV) write_manifest( output_dir=sub_out_dir, inputs={}, cfg={"output_id": "sub_out", "universe_id": "baseline", - "recipe": "echo s", "container_image": None, - "decisions": {}, "code_version": cv, + "recipe": "echo s", + "decisions": {}, "code_version": cv, "env_version": _ENV, "git_sha": "abc", "lc_version": "0.0.1"}, ) return tmp_path @@ -559,13 +562,13 @@ def test_emits_code_repository_entity(self, tmp_path: Path) -> None: out = tmp_path / "results" / "u1" / "foo" out.mkdir(parents=True) (out / "data.txt").write_text("bytes") - cv = code_version(recipe="echo foo", container_image=None, decisions={}) + cv = code_version(recipe="echo foo", decisions={}, env_version=_ENV) write_manifest( output_dir=out, inputs={}, cfg={ "output_id": "foo", "universe_id": "u1", - "recipe": "echo foo", "container_image": None, - "decisions": {}, "code_version": cv, + "recipe": "echo foo", + "decisions": {}, "code_version": cv, "env_version": _ENV, "git_sha": "abc", "git_remote": "https://github.com/dkn16/test-repo", "lc_version": "0.0.1", diff --git a/zensical.toml b/zensical.toml index 0b6be05c..34035572 100644 --- a/zensical.toml +++ b/zensical.toml @@ -14,7 +14,7 @@ nav = [ {"Welcome" = "user/index.md"}, {"Install" = "user/install.md"}, {"Getting Started" = "user/getting-started.md"}, - {"Running on a Cluster" = "user/cluster.md"}, + {"The Environment" = "user/environment.md"}, {"Troubleshooting" = "user/troubleshooting.md"}, {"Glossary" = "user/glossary.md"}, ]}, @@ -24,26 +24,14 @@ nav = [ {"CLI Reference" = [ {"Overview" = "cli/index.md"}, {"lc init" = "cli/init.md"}, + {"lc materialize" = "cli/materialize.md"}, {"lc run" = "cli/run.md"}, {"lc build" = "cli/build.md"}, {"lc status" = "cli/status.md"}, {"lc verify" = "cli/verify.md"}, {"lc export" = "cli/export.md"}, ]}, - {"Python API" = [ - {"Overview" = "api/index.md"}, - {"cli/commands" = "api/cli.md"}, - {"engine/manifest" = "api/manifest.md"}, - {"engine/snakefile" = "api/snakefile.md"}, - {"engine/container" = "api/container.md"}, - {"engine/cloudbuild" = "api/cloudbuild.md"}, - {"engine/status" = "api/status.md"}, - {"engine/verify" = "api/verify.md"}, - {"engine/tree" = "api/tree.md"}, - {"engine/validation" = "api/validation.md"}, - {"engine/dask_cluster" = "api/dask_cluster.md"}, - {"snakemake_executor_plugin_dask" = "api/dask_executor.md"}, - ]}, + {"Python API" = "api/index.md"}, {"Contributing" = [ {"Development Setup" = "contributing/setup.md"}, {"Testing" = "contributing/testing.md"},