From 836cf10ee427cd2e271bb56f9db96c094c2b2f0d Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Mon, 27 Jul 2026 13:29:31 -0400 Subject: [PATCH 1/2] Run the backfill simulation on Modal; the workflow becomes trigger/commit only Post-#107 runs still die on ubuntu-latest: even a single 16-spec levels chunk gets the runner shutdown signal ~5.5 minutes in, before its first simulation completes - one Microsimulation plus the populace/torch import stack no longer fits 7GB, so no chunk size saves the hosted runner. New tools/reform_validation/modal_backfill_app.py runs the IDENTICAL producer path (backfill.py mounted verbatim) in a 64GB Modal container: clones populace at a requested ref, installs the release-exact policyengine-us/-core from the release manifest at runtime, drives the batches with a Volume-backed workdir (partials survive interruptions and resume), and publishes reform_validation_.json to the cd-reform-validation Volume. The workflow keeps its schedule and override-exists guard but never simulates: a tick either spawns the Modal run (fire-and-forget, with a started-marker double-spawn guard) or harvests a finished artifact into the usual auto-merging override PR. Ticks take seconds; timeout drops 330 -> 30 minutes. Requires MODAL_TOKEN_ID / MODAL_TOKEN_SECRET repository secrets (fails with an explicit error until they are set). The Volume is pre-seeded with the Build O artifact, so the first live tick can harvest it directly if #109 has not merged by then. Co-Authored-By: Claude Fable 5 --- .../workflows/reform-validation-backfill.yml | 112 +++++++------ tools/reform_validation/modal_backfill_app.py | 154 ++++++++++++++++++ 2 files changed, 220 insertions(+), 46 deletions(-) create mode 100644 tools/reform_validation/modal_backfill_app.py diff --git a/.github/workflows/reform-validation-backfill.yml b/.github/workflows/reform-validation-backfill.yml index 6a177963..d9f678fc 100644 --- a/.github/workflows/reform-validation-backfill.yml +++ b/.github/workflows/reform-validation-backfill.yml @@ -1,13 +1,27 @@ name: Reform validation backfill -# Decoupled from the release build pipeline: the reform-validation simulation is -# slow (~1-2h) and we never want it to block or break a release. This wakes on a -# timer, checks the current populace-US release, and — only if that release has no -# committed reform_validation.json override yet — reproduces one on the released H5 -# and opens an auto-merging PR. Most runs are a few-second no-op. +# Decoupled from the release build pipeline AND from the runner's hardware: +# the producer needs a full Microsimulation per batch plus the populace/torch +# import stack, which no longer fits GitHub's 7GB ubuntu-latest runner (every +# run since 2026-07-21 died to the OOM shutdown signal, including post-#107 +# chunked runs). The simulation therefore runs on Modal +# (tools/reform_validation/modal_backfill_app.py, 64GB container); this +# workflow is only the trigger/commit layer and every tick finishes in +# seconds: # -# Run it by hand for a specific release via the "Run workflow" button (release_id -# input), or leave it to the schedule. +# tick N: release lacks an override and the Volume has no artifact -> +# deploy + spawn the Modal run (fire-and-forget) and exit. +# tick N+1: artifact is on the Volume -> download, commit the override, +# open an auto-merging PR. (A `started` marker in the Modal +# workdir prevents double-spawns while a run is in flight; the +# Modal workdir checkpoints batch partials, so a re-spawn after +# any failure resumes rather than restarts.) +# +# Requires repository secrets MODAL_TOKEN_ID / MODAL_TOKEN_SECRET for the +# PolicyEngine Modal workspace. +# +# Run by hand for a specific release via the "Run workflow" button +# (release_id input), or leave it to the schedule. on: schedule: @@ -29,6 +43,7 @@ concurrency: env: HF_BASE: https://huggingface.co/datasets/policyengine/populace-us/resolve/main + MODAL_VOLUME: cd-reform-validation jobs: resolve: @@ -59,65 +74,70 @@ jobs: needs: resolve if: needs.resolve.outputs.should_run == 'true' runs-on: ubuntu-latest - timeout-minutes: 330 + timeout-minutes: 30 env: RID: ${{ needs.resolve.outputs.release_id }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} steps: - name: Checkout dashboard uses: actions/checkout@v4 - - name: Checkout populace (producer) - uses: actions/checkout@v4 - with: - repository: PolicyEngine/populace - path: populace - - - name: Read exact build package versions - id: ver - run: | - set -euo pipefail - curl -sL "$HF_BASE/releases/$RID/release_manifest.json" > /tmp/manifest.json - echo "peus=$(python3 -c "import json;print(json.load(open('/tmp/manifest.json'))['build']['built_with_model_package']['version'])")" >> "$GITHUB_OUTPUT" - echo "pecore=$(python3 -c "import json;print(json.load(open('/tmp/manifest.json'))['build']['built_with_core_package']['version'])")" >> "$GITHUB_OUTPUT" - echo "producer=$(git -C populace rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-python@v5 with: python-version: "3.13" - - uses: oven-sh/setup-bun@v2 - - - name: Install producer deps (exact pe-us/pe-core from the release) + - name: Install Modal client run: | - python -m pip install -U pip - # CPU torch first: the default index resolves CUDA wheels (~4GB of - # nvidia-* packages) that waste the hosted runner's disk and memory. - pip install torch --index-url https://download.pytorch.org/whl/cpu - pip install \ - "policyengine-us==${{ steps.ver.outputs.peus }}" \ - "policyengine-core==${{ steps.ver.outputs.pecore }}" \ - tables microdf-python h5py numpy pandas scipy scikit-learn tqdm + set -euo pipefail + if [ -z "$MODAL_TOKEN_ID" ] || [ -z "$MODAL_TOKEN_SECRET" ]; then + echo "::error::MODAL_TOKEN_ID / MODAL_TOKEN_SECRET repository secrets are not set — the simulation runs on Modal and cannot proceed without them." + exit 1 + fi + python -m pip install --quiet modal - - name: Generate reform_validation.json - env: - PYTHONPATH: populace/packages/populace-build/src:populace/packages/populace-frame/src:populace/packages/populace-calibrate/src:populace/packages/populace-fit/src:populace/packages/populace-data/src - RV_PRODUCER_COMMIT: ${{ steps.ver.outputs.producer }} + - name: Harvest artifact or spawn Modal run + id: harvest run: | set -euo pipefail - mkdir -p /tmp/rv - python tools/reform_validation/backfill.py \ - --release-id "$RID" --workdir /tmp/rv \ - --producer-commit "${{ steps.ver.outputs.producer }}" - cp /tmp/rv/reform_validation.json \ - "frontend/lib/populace/reform-overrides/$RID.json" + if modal volume get "$MODAL_VOLUME" "reform_validation_$RID.json" /tmp/rv.json --force; then + echo "::notice::Artifact found on the Volume — committing the override." + echo "have_artifact=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "have_artifact=false" >> "$GITHUB_OUTPUT" + SHA8=$(python3 -c "import hashlib;print(hashlib.sha256('$RID'.encode()).hexdigest()[:8])") + if modal volume ls "$MODAL_VOLUME" "rv_$SHA8" 2>/dev/null | grep -q started; then + echo "::notice::A Modal run for $RID is already in flight (rv_$SHA8/started) — will harvest on a later tick." + exit 0 + fi + PRODUCER_REF=$(curl -sL https://api.github.com/repos/PolicyEngine/populace/commits/main \ + | python3 -c "import sys,json;print(json.load(sys.stdin)['sha'])") + modal deploy tools/reform_validation/modal_backfill_app.py + python3 - "$RID" "$PRODUCER_REF" <<'EOF' + import sys + import modal + + release_id, producer_ref = sys.argv[1], sys.argv[2] + fn = modal.Function.from_name("cd-reform-validation-backfill", "backfill") + call = fn.spawn(release_id, producer_ref) + print(f"::notice::Spawned Modal backfill {call.object_id} for {release_id} " + f"at populace {producer_ref[:9]} — a later tick harvests the artifact.") + EOF + + - uses: oven-sh/setup-bun@v2 + if: steps.harvest.outputs.have_artifact == 'true' - - name: Regenerate overrides map + typecheck + - name: Commit override + typecheck + if: steps.harvest.outputs.have_artifact == 'true' run: | set -euo pipefail + cp /tmp/rv.json "frontend/lib/populace/reform-overrides/$RID.json" node scripts/gen-reform-overrides.mjs cd frontend && bun install --frozen-lockfile && bun run lint - name: Open auto-merge PR + if: steps.harvest.outputs.have_artifact == 'true' env: GH_TOKEN: ${{ github.token }} run: | @@ -131,5 +151,5 @@ jobs: git push -u origin "$BR" gh pr create --base main --head "$BR" \ --title "Backfill reform_validation.json for $RID" \ - --body "Automated backfill by \`.github/workflows/reform-validation-backfill.yml\`. The producer was run on the released \`populace_us_2024.h5\` at the build's exact package versions; provenance is in the file's \`_backfill_note\`. Decoupled from the build pipeline so the slow simulation can't block a release." + --body "Automated backfill by \`.github/workflows/reform-validation-backfill.yml\`. The producer ran on Modal (\`tools/reform_validation/modal_backfill_app.py\`) against the released \`populace_us_2024.h5\` at the build's exact package versions; provenance is in the file's \`_backfill_note\`. The simulation is decoupled from both the build pipeline and the hosted runner, so it can neither block a release nor exhaust the runner." gh pr merge --squash --auto "$BR" || gh pr merge --squash "$BR" diff --git a/tools/reform_validation/modal_backfill_app.py b/tools/reform_validation/modal_backfill_app.py new file mode 100644 index 00000000..ce477528 --- /dev/null +++ b/tools/reform_validation/modal_backfill_app.py @@ -0,0 +1,154 @@ +"""Modal app that runs the reform-validation backfill for a populace release. + +Why Modal: the producer needs one full Microsimulation per batch plus the +populace/torch import stack, which no longer fits GitHub's 7GB ubuntu-latest +runner — every scheduled run since 2026-07-21 died to the runner's OOM +shutdown signal, including post-chunking (#107) runs where even a single +16-spec levels chunk was killed before its first simulation finished. This +app runs the IDENTICAL producer path (tools/reform_validation/backfill.py, +mounted verbatim from the repo checkout at deploy time) in a 64GB container. + +Division of labor with the scheduled workflow +(.github/workflows/reform-validation-backfill.yml): + + GitHub Actions resolve release id -> spawn this app (fire-and- + (trigger/commit layer) forget) -> on a LATER tick, harvest the finished + artifact from the Volume -> commit override -> + auto-merge PR. No simulation ever runs on the + runner, so ticks take seconds. + + This app clone populace at the requested ref, install the + (simulation layer) release-exact policyengine-us/-core from the + release manifest at runtime, drive backfill.py + with a Volume-backed workdir (batch partials + survive interruptions and resume), and publish + reform_validation_.json to the Volume. + +Engine versions are installed at RUNTIME (not baked into the image) because +they are release-exact and change per release; the image carries only the +heavy version-stable dependencies. The producer ref is an argument for the +same reason — no image rebuild per populace commit. + +Manual use: + modal deploy tools/reform_validation/modal_backfill_app.py + python -c " + import modal + fn = modal.Function.from_name('cd-reform-validation-backfill', 'backfill') + print(fn.spawn('', '').object_id)" + +Requires Modal credentials (MODAL_TOKEN_ID / MODAL_TOKEN_SECRET) for the +PolicyEngine workspace. +""" + +from pathlib import Path + +import modal + +app = modal.App("cd-reform-validation-backfill") + +VOLUME_NAME = "cd-reform-validation" +volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True) + +# Heavy, version-stable dependencies only. The release-exact engine +# (policyengine-us / policyengine-core) installs at runtime per release. +image = ( + modal.Image.debian_slim(python_version="3.13") + .apt_install("git") + .pip_install( + "tables", + "microdf-python", + "h5py", + "numpy", + "pandas", + "scipy", + "scikit-learn", + "tqdm", + "requests", + ) + .pip_install("torch", index_url="https://download.pytorch.org/whl/cpu") + .add_local_file( + str(Path(__file__).parent / "backfill.py"), "/root/backfill.py" + ) +) + +PRODUCER_PACKAGES = ( + "populace-build", + "populace-frame", + "populace-calibrate", + "populace-fit", + "populace-data", +) + + +@app.function(image=image, timeout=8 * 3600, memory=65536, cpu=8.0, volumes={"/vol": volume}) +def backfill(release_id: str, producer_ref: str = "main") -> str: + """Produce reform_validation.json for ``release_id`` on the Volume. + + Writes, in order: + /vol/rv_/... durable workdir (H5, partials) + /vol/reform_validation_.json the finished artifact + A ``started`` marker in the workdir lets the workflow avoid double-spawns. + """ + import hashlib + import json + import os + import subprocess + import urllib.request + + subprocess.run( + ["git", "clone", "https://github.com/PolicyEngine/populace", "/opt/populace"], + check=True, + ) + subprocess.run(["git", "-C", "/opt/populace", "checkout", producer_ref], check=True) + producer_commit = subprocess.run( + ["git", "-C", "/opt/populace", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + manifest_url = ( + "https://huggingface.co/datasets/policyengine/populace-us/resolve/main/" + f"releases/{release_id}/release_manifest.json" + ) + with urllib.request.urlopen(manifest_url, timeout=60) as r: + build = json.loads(r.read())["build"] + peus = build["built_with_model_package"]["version"] + pecore = build["built_with_core_package"]["version"] + subprocess.run( + ["pip", "install", "--quiet", f"policyengine-us=={peus}", f"policyengine-core=={pecore}"], + check=True, + ) + print(f"engines installed: policyengine-us {peus} / policyengine-core {pecore}", flush=True) + + workdir = f"/vol/rv_{hashlib.sha256(release_id.encode()).hexdigest()[:8]}" + os.makedirs(workdir, exist_ok=True) + with open(f"{workdir}/started", "w") as f: + f.write(f"{release_id}\nproducer {producer_commit}\n") + volume.commit() + + env = { + **os.environ, + "PYTHONPATH": ":".join( + f"/opt/populace/packages/{p}/src" for p in PRODUCER_PACKAGES + ), + } + proc = subprocess.run( + [ + "python", "/root/backfill.py", + "--release-id", release_id, + "--workdir", workdir, + "--producer-commit", producer_commit, + ], + env=env, + ) + volume.commit() # keep partials even on failure so a re-spawn resumes + proc.check_returncode() + + out = Path(workdir, "reform_validation.json").read_text() + n_rows = len(json.loads(out)["reforms"]) + with open(f"/vol/reform_validation_{release_id}.json", "w") as f: + f.write(out) + volume.commit() + print(f"published {n_rows} rows for {release_id} (producer {producer_commit})", flush=True) + return f"{n_rows} rows" From 128e0d6a15b35b4a9a4a02c0fe98d6204beee91e Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Mon, 3 Aug 2026 11:02:28 -0400 Subject: [PATCH 2/2] Address review: liveness check for spawned runs, gh api for producer ref, workdir cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - spawn_or_wait.py: the spawner records the Modal call id on the Volume and each tick checks the call's real status — a failed run re-spawns (resuming from workdir checkpoints) instead of hiding behind the started marker forever; 24h marker-age fallback when no call id is readable. All four branches exercised against the live Volume. - Producer ref via gh api with the built-in token (no anonymous rate limit). - Harvest deletes the bulky rv_ workdir; the artifact stays as the done-signal until the override merges. Co-Authored-By: Claude Fable 5 --- .../workflows/reform-validation-backfill.yml | 39 +++--- tools/reform_validation/modal_backfill_app.py | 13 +- tools/reform_validation/spawn_or_wait.py | 113 ++++++++++++++++++ 3 files changed, 143 insertions(+), 22 deletions(-) create mode 100644 tools/reform_validation/spawn_or_wait.py diff --git a/.github/workflows/reform-validation-backfill.yml b/.github/workflows/reform-validation-backfill.yml index d9f678fc..d5896be8 100644 --- a/.github/workflows/reform-validation-backfill.yml +++ b/.github/workflows/reform-validation-backfill.yml @@ -12,10 +12,14 @@ name: Reform validation backfill # tick N: release lacks an override and the Volume has no artifact -> # deploy + spawn the Modal run (fire-and-forget) and exit. # tick N+1: artifact is on the Volume -> download, commit the override, -# open an auto-merging PR. (A `started` marker in the Modal -# workdir prevents double-spawns while a run is in flight; the -# Modal workdir checkpoints batch partials, so a re-spawn after -# any failure resumes rather than restarts.) +# open an auto-merging PR. +# +# Liveness (tools/reform_validation/spawn_or_wait.py): the spawner records +# the Modal call id on the Volume next to the app's `started` marker, and +# each tick checks that call's actual status — a failed or vanished run +# re-spawns instead of waiting forever behind a stale marker (age fallback: +# 24h, past the 8h function timeout). Re-spawns are cheap because the Modal +# workdir checkpoints batch partials, so they resume rather than restart. # # Requires repository secrets MODAL_TOKEN_ID / MODAL_TOKEN_SECRET for the # PolicyEngine Modal workspace. @@ -98,32 +102,29 @@ jobs: - name: Harvest artifact or spawn Modal run id: harvest + env: + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail + SHA8=$(python3 -c "import hashlib;print(hashlib.sha256('$RID'.encode()).hexdigest()[:8])") if modal volume get "$MODAL_VOLUME" "reform_validation_$RID.json" /tmp/rv.json --force; then echo "::notice::Artifact found on the Volume — committing the override." echo "have_artifact=true" >> "$GITHUB_OUTPUT" + # The artifact stays on the Volume as the done-signal until the + # override merges; only the bulky workdir (H5 + partials) goes. + modal volume rm "$MODAL_VOLUME" "rv_$SHA8" -r || true exit 0 fi echo "have_artifact=false" >> "$GITHUB_OUTPUT" - SHA8=$(python3 -c "import hashlib;print(hashlib.sha256('$RID'.encode()).hexdigest()[:8])") - if modal volume ls "$MODAL_VOLUME" "rv_$SHA8" 2>/dev/null | grep -q started; then - echo "::notice::A Modal run for $RID is already in flight (rv_$SHA8/started) — will harvest on a later tick." + DECISION=$(python3 tools/reform_validation/spawn_or_wait.py check "$RID") + if [ "$DECISION" = "WAIT" ]; then + echo "::notice::A Modal run for $RID is in flight — will harvest on a later tick." exit 0 fi - PRODUCER_REF=$(curl -sL https://api.github.com/repos/PolicyEngine/populace/commits/main \ - | python3 -c "import sys,json;print(json.load(sys.stdin)['sha'])") + PRODUCER_REF=$(gh api repos/PolicyEngine/populace/commits/main --jq .sha) modal deploy tools/reform_validation/modal_backfill_app.py - python3 - "$RID" "$PRODUCER_REF" <<'EOF' - import sys - import modal - - release_id, producer_ref = sys.argv[1], sys.argv[2] - fn = modal.Function.from_name("cd-reform-validation-backfill", "backfill") - call = fn.spawn(release_id, producer_ref) - print(f"::notice::Spawned Modal backfill {call.object_id} for {release_id} " - f"at populace {producer_ref[:9]} — a later tick harvests the artifact.") - EOF + python3 tools/reform_validation/spawn_or_wait.py spawn "$RID" "$PRODUCER_REF" + echo "::notice::Spawned Modal backfill for $RID at populace ${PRODUCER_REF:0:9} — a later tick harvests the artifact." - uses: oven-sh/setup-bun@v2 if: steps.harvest.outputs.have_artifact == 'true' diff --git a/tools/reform_validation/modal_backfill_app.py b/tools/reform_validation/modal_backfill_app.py index ce477528..704412ee 100644 --- a/tools/reform_validation/modal_backfill_app.py +++ b/tools/reform_validation/modal_backfill_app.py @@ -44,7 +44,8 @@ import modal -app = modal.App("cd-reform-validation-backfill") +APP_NAME = "cd-reform-validation-backfill" +app = modal.App(APP_NAME) VOLUME_NAME = "cd-reform-validation" volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True) @@ -87,12 +88,15 @@ def backfill(release_id: str, producer_ref: str = "main") -> str: Writes, in order: /vol/rv_/... durable workdir (H5, partials) /vol/reform_validation_.json the finished artifact - A ``started`` marker in the workdir lets the workflow avoid double-spawns. + The workdir's ``started`` marker (timestamped) plus the spawner-recorded + ``call_id`` let the workflow distinguish in-flight from dead runs + (spawn_or_wait.py) instead of trusting the marker alone. """ import hashlib import json import os import subprocess + import time import urllib.request subprocess.run( @@ -124,7 +128,10 @@ def backfill(release_id: str, producer_ref: str = "main") -> str: workdir = f"/vol/rv_{hashlib.sha256(release_id.encode()).hexdigest()[:8]}" os.makedirs(workdir, exist_ok=True) with open(f"{workdir}/started", "w") as f: - f.write(f"{release_id}\nproducer {producer_commit}\n") + f.write( + f"{release_id}\nproducer {producer_commit}\n" + f"started_at {int(time.time())}\n" + ) volume.commit() env = { diff --git a/tools/reform_validation/spawn_or_wait.py b/tools/reform_validation/spawn_or_wait.py new file mode 100644 index 00000000..caa5c0ad --- /dev/null +++ b/tools/reform_validation/spawn_or_wait.py @@ -0,0 +1,113 @@ +"""Decide whether a Modal backfill run is in flight, dead, or missing — and +spawn (or re-spawn) it when needed. + +The app writes ``rv_/started`` when a run begins, and the spawner +records the Modal call id at ``rv_/call_id``. A marker alone is not +proof of life: if the producer dies after the marker is written, every later +tick would see "in flight" and nothing would ever retry (PR #112 review). So +``check`` asks Modal for the recorded call's status first, and only falls +back to the marker's age (24h, comfortably past the 8h function timeout) +when no call id is readable. Re-spawning is always safe: the Volume-backed +workdir checkpoints batch partials, so a re-spawn resumes rather than +restarts. + +Usage (from the repo root, Modal credentials in the environment): + python3 tools/reform_validation/spawn_or_wait.py check + prints WAIT or SPAWN on stdout (reasons go to stderr) + python3 tools/reform_validation/spawn_or_wait.py spawn + spawns the backfill and records its call id on the Volume +""" + +import hashlib +import sys +import tempfile +import time +from pathlib import Path + +import modal + +from modal_backfill_app import APP_NAME, VOLUME_NAME + +STALE_AFTER_SECONDS = 24 * 3600 + + +def _sha8(release_id: str) -> str: + return hashlib.sha256(release_id.encode()).hexdigest()[:8] + + +def _read(vol: modal.Volume, path: str) -> str | None: + try: + return b"".join(vol.read_file(path)).decode() + except Exception: + return None + + +def check(release_id: str) -> tuple[str, str]: + vol = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True) + prefix = f"rv_{_sha8(release_id)}" + marker = _read(vol, f"{prefix}/started") + if marker is None: + return "SPAWN", "no run recorded for this release" + + call_id = _read(vol, f"{prefix}/call_id") + if call_id: + call_id = call_id.strip() + try: + modal.FunctionCall.from_id(call_id).get(timeout=0) + except TimeoutError: + return "WAIT", f"call {call_id} is still running" + except Exception as e: + return ( + "SPAWN", + f"call {call_id} failed ({type(e).__name__}: {e}) — " + "re-spawning; the workdir checkpoints make this a resume", + ) + return ( + "SPAWN", + f"call {call_id} finished but no artifact was published — re-spawning", + ) + + started_at = None + for line in marker.splitlines(): + if line.startswith("started_at "): + started_at = float(line.split()[1]) + age = None if started_at is None else time.time() - started_at + if age is not None and age < STALE_AFTER_SECONDS: + return "WAIT", f"no call id readable; marker is {age / 3600:.1f}h old" + return ( + "SPAWN", + "no call id readable and the marker is stale (>24h or unstamped) — " + "re-spawning; the workdir checkpoints make this a resume", + ) + + +def spawn(release_id: str, producer_ref: str) -> None: + fn = modal.Function.from_name(APP_NAME, "backfill") + call = fn.spawn(release_id, producer_ref) + vol = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True) + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp, "call_id") + p.write_text(call.object_id) + with vol.batch_upload(force=True) as batch: + batch.put_file(p, f"rv_{_sha8(release_id)}/call_id") + print( + f"spawned Modal backfill {call.object_id} for {release_id} " + f"at populace {producer_ref[:9]}", + file=sys.stderr, + ) + + +def main() -> None: + mode = sys.argv[1] + if mode == "check": + decision, reason = check(sys.argv[2]) + print(reason, file=sys.stderr) + print(decision) + elif mode == "spawn": + spawn(sys.argv[2], sys.argv[3]) + else: + raise SystemExit(f"unknown mode {mode!r}; use check or spawn") + + +if __name__ == "__main__": + main()