From 240767d95411c41a020e08d4cbc1f62a61ef06df Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 08:12:50 -0700 Subject: [PATCH 01/56] scripts : add Strix host-memory watchdog Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 1 + docs/strix-memory-watchdog.md | 40 ++ scripts/strix_memory_watchdog.py | 664 ++++++++++++++++++++++++++++ tests/CMakeLists.txt | 11 + tests/test_strix_memory_watchdog.py | 406 +++++++++++++++++ 5 files changed, 1122 insertions(+) create mode 100644 docs/strix-memory-watchdog.md create mode 100755 scripts/strix_memory_watchdog.py create mode 100644 tests/test_strix_memory_watchdog.py diff --git a/README.md b/README.md index 3067a7a4e71e..dbf547ce5c5b 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Everything else is upstream `llama.cpp`. The additions currently carried here: | Speculative checkpoints on device | | `llama-server` keeps speculative-decoding checkpoints in device memory instead of copying them to the host | | ROCmFPx quant types | `llama-quantize` types `Q4_0_ROCMFP4`, `Q4_0_ROCMFP4_FAST`, `Q2/Q3/Q6/Q8_0_ROCMFPX` and the `_LEAN`/`_COHERENT`/`_STRIX` recipes | Loads the ROCmFP4 GGUFs published for Strix Halo. CPU codecs plus Vulkan dequant, mat-vec, matmul and integer-dot kernels. Weight formats only: not accepted as KV-cache types | | Repeatable output at depth | | Freed KV cells are zeroed so masked-out rows never carry stale K/V, and the Vulkan radix top-k assigns output slots deterministically | +| Host-memory watchdog | [`scripts/strix_memory_watchdog.py`](docs/strix-memory-watchdog.md) | Runs a command in a process group, requires zero active swap, and stops before host-wide memory reaches the 120 GiB validation ceiling | Every ROCm/HIP change above is guarded on architecture, shape and layout, so other devices see upstream behaviour. Run `--help`, or see [tools/server/README.md](tools/server/README.md), for the full options. diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md new file mode 100644 index 000000000000..4adc0e4be7fe --- /dev/null +++ b/docs/strix-memory-watchdog.md @@ -0,0 +1,40 @@ +# Strix host-memory watchdog + +`scripts/strix_memory_watchdog.py` is an external Linux command wrapper for headless Strix Halo validation. It does not change model loading or cache sizing. It measures host-wide memory from procfs and controls the launched command's process group. + +```sh +./scripts/strix_memory_watchdog.py -- ./build/bin/llama-server +``` + +The wrapper performs these checks and actions: + +- It refuses to launch if `/proc/swaps` contains any active entry. +- It calculates used memory as `MemTotal - MemAvailable`. Linux reports these fields in KiB, so the wrapper multiplies each value by 1024 and keeps all accounting as integer bytes. +- It sends `SIGTERM` to the process group at 116 GiB used. +- It sends `SIGKILL` at 118 GiB used or 30 seconds after `SIGTERM`. +- It sends `SIGKILL` and fails if swap appears or required procfs data becomes unavailable during execution. +- It propagates an unmonitored child exit code. A signal exit uses the shell convention `128 + signal`. + +The 118 GiB emergency threshold leaves a 2 GiB sampling margin below the strict 120 GiB ceiling. The default sample interval is one second. This margin cannot guarantee the ceiling for a workload that can allocate more than 2 GiB between samples. Lower `--emergency-gib` or shorten `--sample-interval-seconds` for such a workload. + +Use `--procfs-root` to select a different procfs mount or a test fixture. `--soft-gib`, `--emergency-gib`, `--grace-seconds`, and `--sample-interval-seconds` override the other defaults. The emergency threshold must remain below 120 GiB. + +The wrapper writes timestamped JSON Lines records to standard error. Preflight, sample, signal, and final records include total, available, used, and peak-used bytes, swap entry count, child status, process-group status, threshold reason, and final classification where applicable. Child standard input, standard output, and standard error are inherited unchanged. + +Exit classifications are authoritative in the final JSON record. Operational failures use these exit codes: + +| Exit code | Classification | +| ---: | --- | +| 2 | procfs or configuration error | +| 3 | swap active at startup or detected during execution | +| 4 | soft threshold reached | +| 5 | emergency threshold reached | +| 6 | soft-threshold grace period expired | +| 7 | process-group signaling or termination failure | +| 127 | command launch failure | + +No model, backend, or ROCm package is required to run the unit tests: + +```sh +python3 tests/test_strix_memory_watchdog.py +``` diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py new file mode 100755 index 000000000000..0dc532c1d027 --- /dev/null +++ b/scripts/strix_memory_watchdog.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import signal +import subprocess +import sys +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import IO, Protocol + + +GIB = 1024**3 +STRICT_CEILING_BYTES = 120 * GIB +DEFAULT_SOFT_BYTES = 116 * GIB +DEFAULT_EMERGENCY_BYTES = 118 * GIB +DEFAULT_GRACE_SECONDS = 30.0 +DEFAULT_SAMPLE_INTERVAL_SECONDS = 1.0 + +EXIT_PROCFS_ERROR = 2 +EXIT_SWAP_ACTIVE = 3 +EXIT_SOFT_LIMIT = 4 +EXIT_EMERGENCY_LIMIT = 5 +EXIT_GRACE_TIMEOUT = 6 +EXIT_SIGNAL_ERROR = 7 +EXIT_LAUNCH_ERROR = 127 + +MEMINFO_VALUE_RE = re.compile(r"([0-9]+) kB") +SWAPS_HEADER = ["Filename", "Type", "Size", "Used", "Priority"] + + +class ProcfsError(RuntimeError): + pass + + +class ProcessGroupError(RuntimeError): + pass + + +class ProcessHandle(Protocol): + pid: int + + def poll(self) -> int | None: + ... + + def wait(self, timeout: float | None = None) -> int: + ... + + +@dataclass(frozen=True) +class HostSnapshot: + total_bytes: int + available_bytes: int + active_swaps: tuple[str, ...] + + @property + def used_bytes(self) -> int: + return self.total_bytes - self.available_bytes + + +@dataclass(frozen=True) +class WatchdogConfig: + command: tuple[str, ...] + procfs_root: Path = Path("/proc") + soft_bytes: int = DEFAULT_SOFT_BYTES + emergency_bytes: int = DEFAULT_EMERGENCY_BYTES + grace_seconds: float = DEFAULT_GRACE_SECONDS + sample_interval_seconds: float = DEFAULT_SAMPLE_INTERVAL_SECONDS + + def validate(self) -> None: + if not self.command: + raise ValueError("a command is required after --") + if self.soft_bytes <= 0: + raise ValueError("soft threshold must be greater than zero") + if self.emergency_bytes <= self.soft_bytes: + raise ValueError("emergency threshold must be greater than soft threshold") + if self.emergency_bytes >= STRICT_CEILING_BYTES: + raise ValueError("emergency threshold must be below 120 GiB") + if not math.isfinite(self.grace_seconds) or self.grace_seconds <= 0: + raise ValueError("grace period must be greater than zero") + if ( + not math.isfinite(self.sample_interval_seconds) + or self.sample_interval_seconds <= 0 + ): + raise ValueError("sample interval must be greater than zero") + + +class ProcfsReader: + def __init__(self, root: Path): + self.root = root + + def _read_text(self, name: str) -> str: + path = self.root / name + try: + return path.read_text(encoding="utf-8") + except OSError as exc: + detail = exc.strerror or str(exc) + raise ProcfsError(f"cannot read {path}: {detail}") from exc + + def read_snapshot(self) -> HostSnapshot: + active_swaps = self._parse_swaps(self._read_text("swaps")) + total_bytes, available_bytes = self._parse_meminfo( + self._read_text("meminfo") + ) + return HostSnapshot(total_bytes, available_bytes, active_swaps) + + @staticmethod + def _parse_meminfo(content: str) -> tuple[int, int]: + values: dict[str, int] = {} + required = {"MemTotal", "MemAvailable"} + for line in content.splitlines(): + key, separator, raw_value = line.partition(":") + if not separator or key not in required: + continue + if key in values: + raise ProcfsError(f"duplicate {key} in meminfo") + match = MEMINFO_VALUE_RE.fullmatch(raw_value.strip()) + if match is None: + raise ProcfsError(f"malformed {key} in meminfo") + values[key] = int(match.group(1)) * 1024 + + missing = sorted(required - values.keys()) + if missing: + raise ProcfsError(f"missing {', '.join(missing)} in meminfo") + if values["MemAvailable"] > values["MemTotal"]: + raise ProcfsError("MemAvailable exceeds MemTotal") + return values["MemTotal"], values["MemAvailable"] + + @staticmethod + def _parse_swaps(content: str) -> tuple[str, ...]: + lines = content.splitlines() + if not lines or lines[0].split() != SWAPS_HEADER: + raise ProcfsError("malformed swaps header") + + entries: list[str] = [] + for line in lines[1:]: + if not line.strip(): + continue + fields = line.split() + if len(fields) != len(SWAPS_HEADER): + raise ProcfsError("malformed swaps entry") + try: + int(fields[2]) + int(fields[3]) + int(fields[4]) + except ValueError as exc: + raise ProcfsError("malformed swaps entry") from exc + entries.append(fields[0]) + return tuple(entries) + + +class AuditLogger: + def __init__( + self, + stream: IO[str], + wall_clock: Callable[[], datetime] | None = None, + ): + self.stream = stream + self.wall_clock = wall_clock or ( + lambda: datetime.now(timezone.utc) + ) + + def emit(self, event: str, **fields: object) -> None: + timestamp = self.wall_clock().astimezone(timezone.utc) + record = { + "timestamp": timestamp.isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ), + "event": event, + **fields, + } + self.stream.write( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" + ) + self.stream.flush() + + +def _child_status(returncode: int | None, started: bool = True) -> str: + if not started: + return "not_started" + if returncode is None: + return "running" + return "signaled" if returncode < 0 else "exited" + + +def _state_fields( + snapshot: HostSnapshot | None, + peak_used_bytes: int | None, + child: ProcessHandle | None, + child_returncode: int | None, + process_group_status: str, + threshold_reason: str, +) -> dict[str, object]: + return { + "total_bytes": snapshot.total_bytes if snapshot else None, + "available_bytes": snapshot.available_bytes if snapshot else None, + "used_bytes": snapshot.used_bytes if snapshot else None, + "swap_entries": len(snapshot.active_swaps) if snapshot else None, + "peak_used_bytes": peak_used_bytes, + "child_pid": child.pid if child else None, + "child_status": _child_status( + child_returncode, started=child is not None + ), + "child_returncode": child_returncode, + "process_group_id": child.pid if child else None, + "process_group_status": process_group_status, + "threshold_reason": threshold_reason, + } + + +def _emit_final( + audit: AuditLogger, + classification: str, + exit_code: int, + reason: str, + snapshot: HostSnapshot | None, + peak_used_bytes: int | None, + child: ProcessHandle | None = None, + child_returncode: int | None = None, + process_group_status: str = "not_created", + error: str | None = None, +) -> int: + fields = _state_fields( + snapshot, + peak_used_bytes, + child, + child_returncode, + process_group_status, + reason, + ) + fields.update(classification=classification, exit_code=exit_code) + if error: + fields["error"] = error + audit.emit("final", **fields) + return exit_code + + +def _signal_process_group(process_group_id: int, signal_number: int) -> str: + try: + os.killpg(process_group_id, signal_number) + except ProcessLookupError: + return "missing" + except OSError as exc: + name = signal.Signals(signal_number).name + detail = exc.strerror or str(exc) + raise ProcessGroupError( + f"cannot send {name} to process group {process_group_id}: {detail}" + ) from exc + return f"{signal.Signals(signal_number).name.lower()}_sent" + + +def _kill_and_finish( + audit: AuditLogger, + child: ProcessHandle, + snapshot: HostSnapshot, + peak_used_bytes: int, + classification: str, + exit_code: int, + reason: str, + signal_group: Callable[[int, int], str], +) -> int: + try: + group_status = signal_group(child.pid, signal.SIGKILL) + except ProcessGroupError as exc: + return _emit_final( + audit, + "signal_error", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "signal_error", + str(exc), + ) + + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + group_status, + reason, + ), + signal="SIGKILL", + ) + try: + child_returncode = child.wait(timeout=5.0) + except subprocess.TimeoutExpired as exc: + return _emit_final( + audit, + "termination_timeout", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "sigkill_timeout", + str(exc), + ) + return _emit_final( + audit, + classification, + exit_code, + reason, + snapshot, + peak_used_bytes, + child, + child_returncode, + group_status, + ) + + +def _monitor_child( + config: WatchdogConfig, + reader: ProcfsReader, + audit: AuditLogger, + child: ProcessHandle, + initial_snapshot: HostSnapshot, + signal_group: Callable[[int, int], str], + monotonic: Callable[[], float], + sleeper: Callable[[float], None], +) -> int: + snapshot = initial_snapshot + peak = snapshot.used_bytes + soft_deadline: float | None = None + + while True: + child_returncode = child.poll() + if child_returncode is not None: + soft_stop = soft_deadline is not None + return _emit_final( + audit, + "soft_limit" if soft_stop else "child_exit", + EXIT_SOFT_LIMIT if soft_stop else ( + 128 - child_returncode + if child_returncode < 0 + else child_returncode + ), + ( + "child exited during soft-threshold grace period" + if soft_stop + else "child exited" + ), + snapshot, + peak, + child, + child_returncode, + "leader_exited", + ) + + now = monotonic() + if soft_deadline is not None and now >= soft_deadline: + return _kill_and_finish( + audit, + child, + snapshot, + peak, + "grace_timeout", + EXIT_GRACE_TIMEOUT, + "soft-threshold grace period expired", + signal_group, + ) + + try: + snapshot = reader.read_snapshot() + except ProcfsError as exc: + return _kill_and_finish( + audit, + child, + snapshot, + peak, + "procfs_error", + EXIT_PROCFS_ERROR, + str(exc), + signal_group, + ) + + peak = max(peak, snapshot.used_bytes) + audit.emit( + "sample", + **_state_fields( + snapshot, peak, child, None, "active", "none" + ), + ) + + if snapshot.active_swaps: + return _kill_and_finish( + audit, + child, + snapshot, + peak, + "swap_appeared", + EXIT_SWAP_ACTIVE, + "active swap appeared during execution", + signal_group, + ) + if snapshot.used_bytes >= config.emergency_bytes: + return _kill_and_finish( + audit, + child, + snapshot, + peak, + "emergency_limit", + EXIT_EMERGENCY_LIMIT, + "used_bytes >= emergency_bytes", + signal_group, + ) + if soft_deadline is None and snapshot.used_bytes >= config.soft_bytes: + try: + group_status = signal_group(child.pid, signal.SIGTERM) + except ProcessGroupError as exc: + return _emit_final( + audit, + "signal_error", + EXIT_SIGNAL_ERROR, + "used_bytes >= soft_bytes", + snapshot, + peak, + child, + child.poll(), + "signal_error", + str(exc), + ) + soft_deadline = now + config.grace_seconds + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak, + child, + child.poll(), + group_status, + "used_bytes >= soft_bytes", + ), + signal="SIGTERM", + grace_deadline_monotonic=soft_deadline, + ) + + sleep_seconds = config.sample_interval_seconds + if soft_deadline is not None: + sleep_seconds = min( + sleep_seconds, + max(0.0, soft_deadline - monotonic()), + ) + sleeper(sleep_seconds) + + +def run_watchdog( + config: WatchdogConfig, + *, + reader: ProcfsReader | None = None, + audit: AuditLogger | None = None, + launcher: Callable[..., ProcessHandle] | None = None, + signal_group: Callable[[int, int], str] | None = None, + monotonic: Callable[[], float] | None = None, + sleeper: Callable[[float], None] | None = None, +) -> int: + config.validate() + reader = reader or ProcfsReader(config.procfs_root) + audit = audit or AuditLogger(sys.stderr) + launcher = launcher or subprocess.Popen + signal_group = signal_group or _signal_process_group + monotonic = monotonic or time.monotonic + sleeper = sleeper or time.sleep + + try: + snapshot = reader.read_snapshot() + except ProcfsError as exc: + return _emit_final( + audit, + "procfs_error", + EXIT_PROCFS_ERROR, + str(exc), + None, + None, + error=str(exc), + ) + + audit.emit( + "preflight", + **_state_fields( + snapshot, + snapshot.used_bytes, + None, + None, + "not_created", + "none", + ), + soft_bytes=config.soft_bytes, + emergency_bytes=config.emergency_bytes, + strict_ceiling_bytes=STRICT_CEILING_BYTES, + ) + + if snapshot.active_swaps: + return _emit_final( + audit, + "startup_swap_active", + EXIT_SWAP_ACTIVE, + "active swap present before command launch", + snapshot, + snapshot.used_bytes, + ) + if snapshot.used_bytes >= config.emergency_bytes: + return _emit_final( + audit, + "startup_emergency_limit", + EXIT_EMERGENCY_LIMIT, + "used_bytes >= emergency_bytes before launch", + snapshot, + snapshot.used_bytes, + ) + if snapshot.used_bytes >= config.soft_bytes: + return _emit_final( + audit, + "startup_soft_limit", + EXIT_SOFT_LIMIT, + "used_bytes >= soft_bytes before launch", + snapshot, + snapshot.used_bytes, + ) + + try: + child = launcher(config.command, start_new_session=True) + except (OSError, ValueError) as exc: + detail = getattr(exc, "strerror", None) or str(exc) + return _emit_final( + audit, + "launch_error", + EXIT_LAUNCH_ERROR, + "command launch failed", + snapshot, + snapshot.used_bytes, + error=detail, + ) + + audit.emit( + "child_started", + **_state_fields( + snapshot, + snapshot.used_bytes, + child, + None, + "active", + "none", + ), + command=list(config.command), + ) + return _monitor_child( + config, + reader, + audit, + child, + snapshot, + signal_group, + monotonic, + sleeper, + ) + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be greater than zero") + return parsed + + +def _positive_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError("value must be greater than zero") + return parsed + + +def parse_args(argv: Sequence[str]) -> WatchdogConfig: + parser = argparse.ArgumentParser( + description=( + "Launch a command in a new process group and stop it before " + "host-wide memory use reaches the 120 GiB Strix validation ceiling." + ) + ) + parser.add_argument( + "--procfs-root", + type=Path, + default=Path("/proc"), + help="procfs root containing meminfo and swaps (default: /proc)", + ) + parser.add_argument( + "--soft-gib", + type=_positive_int, + default=116, + help="send SIGTERM at this many GiB used (default: 116)", + ) + parser.add_argument( + "--emergency-gib", + type=_positive_int, + default=118, + help=( + "send SIGKILL at this many GiB used (default: 118, leaving " + "a 2 GiB sampling margin below 120 GiB)" + ), + ) + parser.add_argument( + "--grace-seconds", + type=_positive_float, + default=DEFAULT_GRACE_SECONDS, + help="maximum time after SIGTERM before SIGKILL (default: 30)", + ) + parser.add_argument( + "--sample-interval-seconds", + type=_positive_float, + default=DEFAULT_SAMPLE_INTERVAL_SECONDS, + help="procfs sampling interval (default: 1)", + ) + parser.add_argument( + "command", + nargs=argparse.REMAINDER, + help="command and arguments, preceded by --", + ) + args = parser.parse_args(argv) + command = tuple(args.command) + if command and command[0] == "--": + command = command[1:] + return WatchdogConfig( + command=command, + procfs_root=args.procfs_root, + soft_bytes=args.soft_gib * GIB, + emergency_bytes=args.emergency_gib * GIB, + grace_seconds=args.grace_seconds, + sample_interval_seconds=args.sample_interval_seconds, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + config = parse_args(argv if argv is not None else sys.argv[1:]) + audit = AuditLogger(sys.stderr) + try: + return run_watchdog(config, audit=audit) + except ValueError as exc: + return _emit_final( + audit, + "configuration_error", + EXIT_PROCFS_ERROR, + "invalid configuration", + None, + None, + error=str(exc), + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ea937784c5a2..e73ea4286197 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -257,6 +257,17 @@ llama_build_and_test(test-chat-template.cpp) # debug tool for chat template differential analysis (not registered as a test, run it manually) llama_build(test-chat-analysis.cpp) llama_build_and_test(test-log.cpp) + +find_package(Python3 3.10 COMPONENTS Interpreter QUIET) +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND Python3_Interpreter_FOUND) + llama_test_cmd( + ${Python3_EXECUTABLE} + NAME test-strix-memory-watchdog + LABEL python + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test_strix_memory_watchdog.py + ) +endif() + llama_build_and_test( test-peg-parser.cpp peg-parser/simple-tokenize.cpp diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py new file mode 100644 index 000000000000..fa4c4aaec969 --- /dev/null +++ b/tests/test_strix_memory_watchdog.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import io +import json +import signal +import subprocess +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[1] + / "scripts" + / "strix_memory_watchdog.py" +) +SPEC = importlib.util.spec_from_file_location( + "strix_memory_watchdog", SCRIPT_PATH +) +assert SPEC is not None +assert SPEC.loader is not None +watchdog = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = watchdog +SPEC.loader.exec_module(watchdog) + + +def snapshot( + used_bytes: int, + *, + total_bytes: int = 200, + active_swaps: tuple[str, ...] = (), +) -> Any: + return watchdog.HostSnapshot( + total_bytes=total_bytes, + available_bytes=total_bytes - used_bytes, + active_swaps=active_swaps, + ) + + +class SequenceReader: + def __init__(self, values: list[Any]): + self.values = values + self.index = 0 + + def read_snapshot(self) -> Any: + index = min(self.index, len(self.values) - 1) + self.index += 1 + value = self.values[index] + if isinstance(value, Exception): + raise value + return value + + +class FakeClock: + def __init__(self): + self.value = 0.0 + + def monotonic(self) -> float: + return self.value + + def sleep(self, seconds: float) -> None: + self.value += seconds + + +class FakeProcess: + def __init__(self, returncode: int | None = None): + self.pid = 4321 + self.returncode = returncode + + def poll(self) -> int | None: + return self.returncode + + def wait(self, timeout: float | None = None) -> int: + if self.returncode is None: + raise subprocess.TimeoutExpired("fake", timeout) + return self.returncode + + +class Harness: + def __init__( + self, + values: list[Any], + process: FakeProcess, + signal_handler: Any | None = None, + ): + self.reader = SequenceReader(values) + self.process = process + self.signal_handler = signal_handler + self.clock = FakeClock() + self.stream = io.StringIO() + self.launched = False + self.signals: list[int] = [] + fixed_time = datetime(2026, 1, 1, tzinfo=timezone.utc) + self.audit = watchdog.AuditLogger( + self.stream, wall_clock=lambda: fixed_time + ) + + def launcher(self, command: tuple[str, ...], **kwargs: Any) -> FakeProcess: + self.launched = True + self.command = command + self.launch_kwargs = kwargs + return self.process + + def signal_group(self, process_group_id: int, signal_number: int) -> str: + self.signals.append(signal_number) + if self.signal_handler is not None: + self.signal_handler(self.process, signal_number) + return f"{signal.Signals(signal_number).name.lower()}_sent" + + def run(self, **overrides: Any) -> int: + config = watchdog.WatchdogConfig( + command=("fake-command",), + soft_bytes=100, + emergency_bytes=150, + grace_seconds=2, + sample_interval_seconds=1, + **overrides, + ) + return watchdog.run_watchdog( + config, + reader=self.reader, + audit=self.audit, + launcher=self.launcher, + signal_group=self.signal_group, + monotonic=self.clock.monotonic, + sleeper=self.clock.sleep, + ) + + def records(self) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in self.stream.getvalue().splitlines() + ] + + +class TestProcfsParsing(unittest.TestCase): + def test_parses_meminfo_as_integer_bytes_and_allows_zero_swap(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "meminfo").write_text( + "MemTotal: 131072 kB\n" + "MemFree: 4096 kB\n" + "MemAvailable: 32768 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + + result = watchdog.ProcfsReader(root).read_snapshot() + + self.assertEqual(result.total_bytes, 131072 * 1024) + self.assertEqual(result.available_bytes, 32768 * 1024) + self.assertEqual(result.used_bytes, 98304 * 1024) + self.assertEqual(result.active_swaps, ()) + + def test_rejects_active_swap_entry(self) -> None: + content = ( + "Filename Type Size Used Priority\n" + "/swapfile file 1048572 0 -2\n" + ) + self.assertEqual( + watchdog.ProcfsReader._parse_swaps(content), + ("/swapfile",), + ) + + def test_rejects_malformed_or_missing_procfs_data(self) -> None: + with self.assertRaisesRegex( + watchdog.ProcfsError, "malformed MemAvailable" + ): + watchdog.ProcfsReader._parse_meminfo( + "MemTotal: 10 kB\nMemAvailable: unknown\n" + ) + with self.assertRaisesRegex( + watchdog.ProcfsError, "missing MemAvailable" + ): + watchdog.ProcfsReader._parse_meminfo("MemTotal: 10 kB\n") + with self.assertRaisesRegex( + watchdog.ProcfsError, "malformed swaps header" + ): + watchdog.ProcfsReader._parse_swaps("") + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex( + watchdog.ProcfsError, "cannot read" + ): + watchdog.ProcfsReader( + Path(temp_dir) + ).read_snapshot() + + +class TestWatchdogBehavior(unittest.TestCase): + def test_configuration_rejects_non_finite_timing(self) -> None: + config = watchdog.WatchdogConfig( + command=("fake-command",), + grace_seconds=float("nan"), + ) + with self.assertRaisesRegex(ValueError, "grace period"): + config.validate() + + def test_cli_fixture_launches_command_and_propagates_exit(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "meminfo").write_text( + "MemTotal: 131072 kB\nMemAvailable: 65536 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--sample-interval-seconds", + "0.01", + "--", + sys.executable, + "-c", + "raise SystemExit(23)", + ], + capture_output=True, + check=False, + text=True, + ) + + self.assertEqual(result.returncode, 23) + records = [ + json.loads(line) for line in result.stderr.splitlines() + ] + self.assertEqual(records[-1]["classification"], "child_exit") + self.assertEqual(records[-1]["child_returncode"], 23) + + def test_zero_swap_gate_launches_and_propagates_child_exit(self) -> None: + harness = Harness([snapshot(50)], FakeProcess(returncode=37)) + + result = harness.run() + + self.assertEqual(result, 37) + self.assertTrue(harness.launched) + self.assertTrue(harness.launch_kwargs["start_new_session"]) + final = harness.records()[-1] + self.assertEqual(final["classification"], "child_exit") + self.assertEqual(final["total_bytes"], 200) + self.assertEqual(final["available_bytes"], 150) + self.assertEqual(final["used_bytes"], 50) + self.assertEqual(final["peak_used_bytes"], 50) + self.assertEqual(final["child_status"], "exited") + self.assertEqual(final["process_group_status"], "leader_exited") + + def test_signaled_child_exit_uses_shell_exit_convention(self) -> None: + harness = Harness( + [snapshot(50)], + FakeProcess(returncode=-signal.SIGTERM), + ) + + result = harness.run() + + self.assertEqual(result, 128 + signal.SIGTERM) + + def test_active_swap_rejects_startup_without_launch(self) -> None: + harness = Harness( + [snapshot(50, active_swaps=("/swapfile",))], + FakeProcess(), + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_SWAP_ACTIVE) + self.assertFalse(harness.launched) + self.assertEqual( + harness.records()[-1]["classification"], + "startup_swap_active", + ) + + def test_soft_limit_sends_sigterm(self) -> None: + def exit_on_term(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGTERM: + process.returncode = -signal.SIGTERM + + harness = Harness( + [snapshot(50), snapshot(110)], + FakeProcess(), + signal_handler=exit_on_term, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_SOFT_LIMIT) + self.assertEqual(harness.signals, [signal.SIGTERM]) + self.assertEqual( + harness.records()[-1]["classification"], "soft_limit" + ) + + def test_emergency_limit_sends_sigkill(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), snapshot(160)], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_EMERGENCY_LIMIT) + self.assertEqual(harness.signals, [signal.SIGKILL]) + self.assertEqual( + harness.records()[-1]["classification"], "emergency_limit" + ) + + def test_grace_timeout_escalates_to_sigkill(self) -> None: + def ignore_term(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), snapshot(110)], + FakeProcess(), + signal_handler=ignore_term, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_GRACE_TIMEOUT) + self.assertEqual( + harness.signals, + [signal.SIGTERM, signal.SIGKILL], + ) + self.assertEqual(harness.clock.value, 2.0) + self.assertEqual( + harness.records()[-1]["classification"], "grace_timeout" + ) + + def test_swap_appearing_during_execution_kills_group(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [ + snapshot(50), + snapshot(60, active_swaps=("/swapfile",)), + ], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_SWAP_ACTIVE) + self.assertEqual(harness.signals, [signal.SIGKILL]) + self.assertEqual( + harness.records()[-1]["classification"], "swap_appeared" + ) + + def test_runtime_procfs_error_kills_group(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), watchdog.ProcfsError("missing meminfo")], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_PROCFS_ERROR) + self.assertEqual(harness.signals, [signal.SIGKILL]) + self.assertEqual( + harness.records()[-1]["classification"], "procfs_error" + ) + + def test_launch_failure_is_explicit(self) -> None: + harness = Harness([snapshot(50)], FakeProcess()) + + def fail_launch( + command: tuple[str, ...], **kwargs: Any + ) -> FakeProcess: + raise FileNotFoundError(2, "No such file or directory") + + harness.launcher = fail_launch + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_LAUNCH_ERROR) + self.assertEqual( + harness.records()[-1]["classification"], "launch_error" + ) + + +if __name__ == "__main__": + unittest.main() From 41dbf04fbabc5c94d6cda4930c7f7b92361e7823 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 08:20:14 -0700 Subject: [PATCH 02/56] scripts : clean up watchdog process group on signals Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 3 + scripts/strix_memory_watchdog.py | 309 +++++++++++++++++++++++----- tests/test_strix_memory_watchdog.py | 150 ++++++++++++++ 3 files changed, 407 insertions(+), 55 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 4adc0e4be7fe..8d6d264b141b 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -13,6 +13,8 @@ The wrapper performs these checks and actions: - It sends `SIGTERM` to the process group at 116 GiB used. - It sends `SIGKILL` at 118 GiB used or 30 seconds after `SIGTERM`. - It sends `SIGKILL` and fails if swap appears or required procfs data becomes unavailable during execution. +- It forwards wrapper `SIGINT` or `SIGTERM` to the process group, waits the configured grace period, then sends `SIGKILL` if any group member remains. +- It applies the same bounded process-group cleanup if an unexpected post-launch error occurs. - It propagates an unmonitored child exit code. A signal exit uses the shell convention `128 + signal`. The 118 GiB emergency threshold leaves a 2 GiB sampling margin below the strict 120 GiB ceiling. The default sample interval is one second. This margin cannot guarantee the ceiling for a workload that can allocate more than 2 GiB between samples. Lower `--emergency-gib` or shorten `--sample-interval-seconds` for such a workload. @@ -31,6 +33,7 @@ Exit classifications are authoritative in the final JSON record. Operational fai | 5 | emergency threshold reached | | 6 | soft-threshold grace period expired | | 7 | process-group signaling or termination failure | +| 70 | unexpected post-launch error | | 127 | command launch failure | No model, backend, or ROCm package is required to run the unit tests: diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index 0dc532c1d027..d8738d5041fd 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -31,10 +31,12 @@ EXIT_EMERGENCY_LIMIT = 5 EXIT_GRACE_TIMEOUT = 6 EXIT_SIGNAL_ERROR = 7 +EXIT_INTERNAL_ERROR = 70 EXIT_LAUNCH_ERROR = 127 MEMINFO_VALUE_RE = re.compile(r"([0-9]+) kB") SWAPS_HEADER = ["Filename", "Type", "Size", "Used", "Priority"] +PARENT_SIGNALS = (signal.SIGINT, signal.SIGTERM) class ProcfsError(RuntimeError): @@ -45,6 +47,12 @@ class ProcessGroupError(RuntimeError): pass +class ParentSignal(RuntimeError): + def __init__(self, signal_number: int): + self.signal_number = signal_number + super().__init__(signal.Signals(signal_number).name) + + class ProcessHandle(Protocol): pid: int @@ -66,6 +74,12 @@ def used_bytes(self) -> int: return self.total_bytes - self.available_bytes +@dataclass +class RuntimeState: + snapshot: HostSnapshot + peak_used_bytes: int + + @dataclass(frozen=True) class WatchdogConfig: command: tuple[str, ...] @@ -257,6 +271,39 @@ def _signal_process_group(process_group_id: int, signal_number: int) -> str: return f"{signal.Signals(signal_number).name.lower()}_sent" +def _process_group_alive(process_group_id: int) -> bool: + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except OSError as exc: + detail = exc.strerror or str(exc) + raise ProcessGroupError( + f"cannot inspect process group {process_group_id}: {detail}" + ) from exc + return True + + +def _raise_parent_signal(signal_number: int, _frame: object) -> None: + raise ParentSignal(signal_number) + + +def _set_parent_signal_handlers( + handler: signal.Handlers, +) -> dict[int, signal.Handlers]: + previous: dict[int, signal.Handlers] = {} + for signal_number in PARENT_SIGNALS: + previous[signal_number] = signal.signal(signal_number, handler) + return previous + + +def _restore_parent_signal_handlers( + previous: dict[int, signal.Handlers], +) -> None: + for signal_number, handler in previous.items(): + signal.signal(signal_number, handler) + + def _kill_and_finish( audit: AuditLogger, child: ProcessHandle, @@ -323,18 +370,105 @@ def _kill_and_finish( ) +def _graceful_cleanup( + audit: AuditLogger, + child: ProcessHandle, + snapshot: HostSnapshot, + peak_used_bytes: int, + classification: str, + exit_code: int, + reason: str, + graceful_signal: int, + grace_seconds: float, + signal_group: Callable[[int, int], str], + group_alive: Callable[[int], bool], + monotonic: Callable[[], float], + sleeper: Callable[[float], None], + error: str | None = None, +) -> int: + signal_events: list[tuple[int, str]] = [] + try: + group_status = signal_group(child.pid, graceful_signal) + signal_events.append((graceful_signal, group_status)) + deadline = monotonic() + grace_seconds + while monotonic() < deadline: + child.poll() + if not group_alive(child.pid): + break + sleeper(min(0.05, deadline - monotonic())) + child.poll() + if group_alive(child.pid): + group_status = signal_group(child.pid, signal.SIGKILL) + signal_events.append((signal.SIGKILL, group_status)) + except ProcessGroupError as exc: + return _emit_final( + audit, + "signal_error", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "signal_error", + str(exc), + ) + + child_returncode = child.poll() + if child_returncode is None: + try: + child_returncode = child.wait(timeout=5.0) + except subprocess.TimeoutExpired as exc: + return _emit_final( + audit, + "termination_timeout", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "termination_timeout", + str(exc), + ) + + for signal_number, status in signal_events: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child_returncode, + status, + reason, + ), + signal=signal.Signals(signal_number).name, + ) + return _emit_final( + audit, + classification, + exit_code, + reason, + snapshot, + peak_used_bytes, + child, + child_returncode, + signal_events[-1][1], + error, + ) + + def _monitor_child( config: WatchdogConfig, reader: ProcfsReader, audit: AuditLogger, child: ProcessHandle, - initial_snapshot: HostSnapshot, + state: RuntimeState, signal_group: Callable[[int, int], str], monotonic: Callable[[], float], sleeper: Callable[[float], None], ) -> int: - snapshot = initial_snapshot - peak = snapshot.used_bytes soft_deadline: float | None = None while True: @@ -354,8 +488,8 @@ def _monitor_child( if soft_stop else "child exited" ), - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, child, child_returncode, "leader_exited", @@ -366,8 +500,8 @@ def _monitor_child( return _kill_and_finish( audit, child, - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, "grace_timeout", EXIT_GRACE_TIMEOUT, "soft-threshold grace period expired", @@ -375,50 +509,60 @@ def _monitor_child( ) try: - snapshot = reader.read_snapshot() + state.snapshot = reader.read_snapshot() except ProcfsError as exc: return _kill_and_finish( audit, child, - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, "procfs_error", EXIT_PROCFS_ERROR, str(exc), signal_group, ) - peak = max(peak, snapshot.used_bytes) + state.peak_used_bytes = max( + state.peak_used_bytes, state.snapshot.used_bytes + ) audit.emit( "sample", **_state_fields( - snapshot, peak, child, None, "active", "none" + state.snapshot, + state.peak_used_bytes, + child, + None, + "active", + "none", ), ) - if snapshot.active_swaps: + if state.snapshot.active_swaps: return _kill_and_finish( audit, child, - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, "swap_appeared", EXIT_SWAP_ACTIVE, "active swap appeared during execution", signal_group, ) - if snapshot.used_bytes >= config.emergency_bytes: + if state.snapshot.used_bytes >= config.emergency_bytes: return _kill_and_finish( audit, child, - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, "emergency_limit", EXIT_EMERGENCY_LIMIT, "used_bytes >= emergency_bytes", signal_group, ) - if soft_deadline is None and snapshot.used_bytes >= config.soft_bytes: + if ( + soft_deadline is None + and state.snapshot.used_bytes >= config.soft_bytes + ): try: group_status = signal_group(child.pid, signal.SIGTERM) except ProcessGroupError as exc: @@ -427,8 +571,8 @@ def _monitor_child( "signal_error", EXIT_SIGNAL_ERROR, "used_bytes >= soft_bytes", - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, child, child.poll(), "signal_error", @@ -438,8 +582,8 @@ def _monitor_child( audit.emit( "process_group_signal", **_state_fields( - snapshot, - peak, + state.snapshot, + state.peak_used_bytes, child, child.poll(), group_status, @@ -465,6 +609,7 @@ def run_watchdog( audit: AuditLogger | None = None, launcher: Callable[..., ProcessHandle] | None = None, signal_group: Callable[[int, int], str] | None = None, + group_alive: Callable[[int], bool] | None = None, monotonic: Callable[[], float] | None = None, sleeper: Callable[[float], None] | None = None, ) -> int: @@ -473,6 +618,7 @@ def run_watchdog( audit = audit or AuditLogger(sys.stderr) launcher = launcher or subprocess.Popen signal_group = signal_group or _signal_process_group + group_alive = group_alive or _process_group_alive monotonic = monotonic or time.monotonic sleeper = sleeper or time.sleep @@ -532,42 +678,95 @@ def run_watchdog( snapshot.used_bytes, ) + previous_mask: set[signal.Signals] | None = signal.pthread_sigmask( + signal.SIG_BLOCK, PARENT_SIGNALS + ) + previous_handlers: dict[int, signal.Handlers] = {} + child: ProcessHandle | None = None + state = RuntimeState(snapshot, snapshot.used_bytes) try: - child = launcher(config.command, start_new_session=True) - except (OSError, ValueError) as exc: - detail = getattr(exc, "strerror", None) or str(exc) - return _emit_final( + try: + child = launcher(config.command, start_new_session=True) + except (OSError, ValueError) as exc: + detail = getattr(exc, "strerror", None) or str(exc) + return _emit_final( + audit, + "launch_error", + EXIT_LAUNCH_ERROR, + "command launch failed", + snapshot, + snapshot.used_bytes, + error=detail, + ) + + previous_handlers = _set_parent_signal_handlers( + _raise_parent_signal + ) + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + previous_mask = None + audit.emit( + "child_started", + **_state_fields( + state.snapshot, + state.peak_used_bytes, + child, + None, + "active", + "none", + ), + command=list(config.command), + ) + return _monitor_child( + config, + reader, audit, - "launch_error", - EXIT_LAUNCH_ERROR, - "command launch failed", - snapshot, - snapshot.used_bytes, - error=detail, + child, + state, + signal_group, + monotonic, + sleeper, ) - - audit.emit( - "child_started", - **_state_fields( - snapshot, - snapshot.used_bytes, + except ParentSignal as exc: + _set_parent_signal_handlers(signal.SIG_IGN) + signal_name = signal.Signals(exc.signal_number).name + return _graceful_cleanup( + audit, child, - None, - "active", - "none", - ), - command=list(config.command), - ) - return _monitor_child( - config, - reader, - audit, - child, - snapshot, - signal_group, - monotonic, - sleeper, - ) + state.snapshot, + state.peak_used_bytes, + "parent_signal", + 128 + exc.signal_number, + f"wrapper received {signal_name}", + exc.signal_number, + config.grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + ) + except Exception as exc: + _set_parent_signal_handlers(signal.SIG_IGN) + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "internal_error", + EXIT_INTERNAL_ERROR, + "unexpected post-launch exception", + signal.SIGTERM, + config.grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + f"{type(exc).__name__}: {exc}", + ) + finally: + if previous_mask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + if previous_handlers: + _restore_parent_signal_handlers(previous_handlers) def _positive_int(value: str) -> int: diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index fa4c4aaec969..c0dbc9c1fcb3 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -5,10 +5,12 @@ import importlib.util import io import json +import os import signal import subprocess import sys import tempfile +import time import unittest from datetime import datetime, timezone from pathlib import Path @@ -113,6 +115,9 @@ def signal_group(self, process_group_id: int, signal_number: int) -> str: self.signal_handler(self.process, signal_number) return f"{signal.Signals(signal_number).name.lower()}_sent" + def group_alive(self, process_group_id: int) -> bool: + return self.process.returncode is None + def run(self, **overrides: Any) -> int: config = watchdog.WatchdogConfig( command=("fake-command",), @@ -128,6 +133,7 @@ def run(self, **overrides: Any) -> int: audit=self.audit, launcher=self.launcher, signal_group=self.signal_group, + group_alive=self.group_alive, monotonic=self.clock.monotonic, sleeper=self.clock.sleep, ) @@ -196,6 +202,128 @@ def test_rejects_malformed_or_missing_procfs_data(self) -> None: class TestWatchdogBehavior(unittest.TestCase): + @staticmethod + def _process_is_running(process_id: int) -> bool: + result = subprocess.run( + ["ps", "-o", "stat=", "-p", str(process_id)], + capture_output=True, + check=False, + text=True, + ) + return result.returncode == 0 and not result.stdout.lstrip().startswith( + "Z" + ) + + def test_parent_signals_leave_no_child_or_grandchild(self) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGINT,signal.SIG_IGN);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "grandchild=os.fork();" + "\nif grandchild == 0:\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n');" + " time.sleep(30)\n" + ) + for signal_number in (signal.SIGINT, signal.SIGTERM): + with self.subTest(signal=signal.Signals(signal_number).name): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pids" + (root / "meminfo").write_text( + "MemTotal: 131072 kB\n" + "MemAvailable: 65536 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + audit_path = root / "audit.jsonl" + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--grace-seconds", + "0.2", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + self.fail( + "child process group did not start" + ) + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_file.read_text( + encoding="utf-8" + ).split() + ) + time.sleep(0.05) + wrapper.send_signal(signal_number) + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + self.assertEqual( + wrapper.returncode, 128 + signal_number + ) + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + self.assertEqual( + records[-1]["classification"], "parent_signal" + ) + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual( + forwarded[0], + signal.Signals(signal_number).name, + ) + self.assertEqual(forwarded[-1], "SIGKILL") + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse( + self._process_is_running(process_id) + ) + def test_configuration_rejects_non_finite_timing(self) -> None: config = watchdog.WatchdogConfig( command=("fake-command",), @@ -384,6 +512,28 @@ def exit_on_kill(process: FakeProcess, signal_number: int) -> None: harness.records()[-1]["classification"], "procfs_error" ) + def test_unexpected_monitor_error_cleans_up_process_group(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), RuntimeError("unexpected")], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_INTERNAL_ERROR) + self.assertEqual( + harness.signals, + [signal.SIGTERM, signal.SIGKILL], + ) + final = harness.records()[-1] + self.assertEqual(final["classification"], "internal_error") + self.assertIn("RuntimeError: unexpected", final["error"]) + def test_launch_failure_is_explicit(self) -> None: harness = Harness([snapshot(50)], FakeProcess()) From da5ce95a8c08a9e01c01cd3519de188c8dca2512 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 08:37:36 -0700 Subject: [PATCH 03/56] scripts : preserve child signals and monitor descendants Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 3 +- scripts/strix_memory_watchdog.py | 129 ++++++++++++----- tests/test_strix_memory_watchdog.py | 206 +++++++++++++++++++++++++--- 3 files changed, 282 insertions(+), 56 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 8d6d264b141b..5a6558f0a9c0 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -14,6 +14,7 @@ The wrapper performs these checks and actions: - It sends `SIGKILL` at 118 GiB used or 30 seconds after `SIGTERM`. - It sends `SIGKILL` and fails if swap appears or required procfs data becomes unavailable during execution. - It forwards wrapper `SIGINT` or `SIGTERM` to the process group, waits the configured grace period, then sends `SIGKILL` if any group member remains. +- It checks the process group after the direct child exits and cleans up remaining descendants before returning the child's classification. - It applies the same bounded process-group cleanup if an unexpected post-launch error occurs. - It propagates an unmonitored child exit code. A signal exit uses the shell convention `128 + signal`. @@ -21,7 +22,7 @@ The 118 GiB emergency threshold leaves a 2 GiB sampling margin below the strict Use `--procfs-root` to select a different procfs mount or a test fixture. `--soft-gib`, `--emergency-gib`, `--grace-seconds`, and `--sample-interval-seconds` override the other defaults. The emergency threshold must remain below 120 GiB. -The wrapper writes timestamped JSON Lines records to standard error. Preflight, sample, signal, and final records include total, available, used, and peak-used bytes, swap entry count, child status, process-group status, threshold reason, and final classification where applicable. Child standard input, standard output, and standard error are inherited unchanged. +The wrapper writes timestamped JSON Lines records to standard error. Preflight, sample, signal, and final records include total, available, used, and peak-used bytes, swap entry count, child status, process-group status, threshold reason, and final classification where applicable. Signal records are written immediately after each process-group signal. Child standard input, standard output, and standard error are inherited unchanged. Exit classifications are authoritative in the final JSON record. Operational failures use these exit codes: diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index d8738d5041fd..79d7a0013e77 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -378,18 +378,32 @@ def _graceful_cleanup( classification: str, exit_code: int, reason: str, - graceful_signal: int, + graceful_signal: int | None, grace_seconds: float, signal_group: Callable[[int, int], str], group_alive: Callable[[int], bool], monotonic: Callable[[], float], sleeper: Callable[[float], None], + process_group_status: str = "active", error: str | None = None, ) -> int: - signal_events: list[tuple[int, str]] = [] try: - group_status = signal_group(child.pid, graceful_signal) - signal_events.append((graceful_signal, group_status)) + if graceful_signal is not None: + process_group_status = signal_group( + child.pid, graceful_signal + ) + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + process_group_status, + reason, + ), + signal=signal.Signals(graceful_signal).name, + ) deadline = monotonic() + grace_seconds while monotonic() < deadline: child.poll() @@ -398,8 +412,21 @@ def _graceful_cleanup( sleeper(min(0.05, deadline - monotonic())) child.poll() if group_alive(child.pid): - group_status = signal_group(child.pid, signal.SIGKILL) - signal_events.append((signal.SIGKILL, group_status)) + process_group_status = signal_group( + child.pid, signal.SIGKILL + ) + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + process_group_status, + reason, + ), + signal="SIGKILL", + ) except ProcessGroupError as exc: return _emit_final( audit, @@ -432,19 +459,6 @@ def _graceful_cleanup( str(exc), ) - for signal_number, status in signal_events: - audit.emit( - "process_group_signal", - **_state_fields( - snapshot, - peak_used_bytes, - child, - child_returncode, - status, - reason, - ), - signal=signal.Signals(signal_number).name, - ) return _emit_final( audit, classification, @@ -454,7 +468,7 @@ def _graceful_cleanup( peak_used_bytes, child, child_returncode, - signal_events[-1][1], + process_group_status, error, ) @@ -466,6 +480,7 @@ def _monitor_child( child: ProcessHandle, state: RuntimeState, signal_group: Callable[[int, int], str], + group_alive: Callable[[int], bool], monotonic: Callable[[], float], sleeper: Callable[[float], None], ) -> int: @@ -475,19 +490,50 @@ def _monitor_child( child_returncode = child.poll() if child_returncode is not None: soft_stop = soft_deadline is not None + classification = "soft_limit" if soft_stop else "child_exit" + exit_code = EXIT_SOFT_LIMIT if soft_stop else ( + 128 - child_returncode + if child_returncode < 0 + else child_returncode + ) + reason = ( + "child exited during soft-threshold grace period" + if soft_stop + else "child exited" + ) + if group_alive(child.pid): + grace_seconds = config.grace_seconds + graceful_signal: int | None = signal.SIGTERM + group_status = "active" + if soft_stop: + grace_seconds = max( + 0.0, soft_deadline - monotonic() + ) + graceful_signal = None + group_status = "sigterm_sent" + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + classification, + exit_code, + ( + f"{reason}; process group members still running" + ), + graceful_signal, + grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + group_status, + ) return _emit_final( audit, - "soft_limit" if soft_stop else "child_exit", - EXIT_SOFT_LIMIT if soft_stop else ( - 128 - child_returncode - if child_returncode < 0 - else child_returncode - ), - ( - "child exited during soft-threshold grace period" - if soft_stop - else "child exited" - ), + classification, + exit_code, + reason, state.snapshot, state.peak_used_bytes, child, @@ -678,15 +724,25 @@ def run_watchdog( snapshot.used_bytes, ) - previous_mask: set[signal.Signals] | None = signal.pthread_sigmask( + previous_mask = signal.pthread_sigmask( signal.SIG_BLOCK, PARENT_SIGNALS ) + mask_restored = False previous_handlers: dict[int, signal.Handlers] = {} child: ProcessHandle | None = None state = RuntimeState(snapshot, snapshot.used_bytes) try: + launch_mask = previous_mask + + def restore_child_signal_mask() -> None: + signal.pthread_sigmask(signal.SIG_SETMASK, launch_mask) + try: - child = launcher(config.command, start_new_session=True) + child = launcher( + config.command, + start_new_session=True, + preexec_fn=restore_child_signal_mask, + ) except (OSError, ValueError) as exc: detail = getattr(exc, "strerror", None) or str(exc) return _emit_final( @@ -703,7 +759,7 @@ def run_watchdog( _raise_parent_signal ) signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) - previous_mask = None + mask_restored = True audit.emit( "child_started", **_state_fields( @@ -723,6 +779,7 @@ def run_watchdog( child, state, signal_group, + group_alive, monotonic, sleeper, ) @@ -760,10 +817,10 @@ def run_watchdog( group_alive, monotonic, sleeper, - f"{type(exc).__name__}: {exc}", + error=f"{type(exc).__name__}: {exc}", ) finally: - if previous_mask is not None: + if not mask_restored: signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) if previous_handlers: _restore_parent_signal_handlers(previous_handlers) diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index c0dbc9c1fcb3..2e59eec27512 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -214,6 +214,17 @@ def _process_is_running(process_id: int) -> bool: "Z" ) + @staticmethod + def _write_procfs_fixture(root: Path) -> None: + (root / "meminfo").write_text( + "MemTotal: 131072 kB\nMemAvailable: 65536 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + def test_parent_signals_leave_no_child_or_grandchild(self) -> None: child_code = ( "import os,signal,sys,time;" @@ -232,15 +243,7 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) pid_file = root / "pids" - (root / "meminfo").write_text( - "MemTotal: 131072 kB\n" - "MemAvailable: 65536 kB\n", - encoding="utf-8", - ) - (root / "swaps").write_text( - "Filename Type Size Used Priority\n", - encoding="utf-8", - ) + self._write_procfs_fixture(root) audit_path = root / "audit.jsonl" with audit_path.open("w", encoding="utf-8") as audit: wrapper = subprocess.Popen( @@ -303,16 +306,29 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: self.assertEqual( records[-1]["classification"], "parent_signal" ) - forwarded = [ - record["signal"] + signal_records = [ + record for record in records if record["event"] == "process_group_signal" ] + forwarded = [ + record["signal"] for record in signal_records + ] self.assertEqual( forwarded[0], signal.Signals(signal_number).name, ) self.assertEqual(forwarded[-1], "SIGKILL") + self.assertEqual( + signal_records[0]["child_status"], "running" + ) + self.assertIsNone( + signal_records[0]["child_returncode"] + ) + self.assertLess( + signal_records[0]["timestamp"], + signal_records[-1]["timestamp"], + ) for process_id in (child_pid, grandchild_pid): deadline = time.monotonic() + 2 while ( @@ -324,6 +340,165 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: self._process_is_running(process_id) ) + def test_child_sigterm_handler_exits_without_escalation(self) -> None: + child_code = ( + "import os,signal,sys,time\n" + "def stop(_signal,_frame):\n" + " open(sys.argv[2],'w').write('handled\\n')\n" + " raise SystemExit(0)\n" + "signal.signal(signal.SIGTERM,stop)\n" + "open(sys.argv[1],'w').write(f'{os.getpid()}\\n')\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + ready_path = root / "ready" + handled_path = root / "handled" + audit_path = root / "audit.jsonl" + self._write_procfs_fixture(root) + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--grace-seconds", + "0.5", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(ready_path), + str(handled_path), + ], + stderr=audit, + text=True, + ) + deadline = time.monotonic() + 5 + while not ready_path.exists(): + if time.monotonic() >= deadline: + wrapper.kill() + wrapper.wait(timeout=5) + self.fail("SIGTERM child did not become ready") + time.sleep(0.01) + child_pid = int( + ready_path.read_text(encoding="utf-8").strip() + ) + try: + wrapper.send_signal(signal.SIGTERM) + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual(wrapper.returncode, 128 + signal.SIGTERM) + self.assertTrue(handled_path.exists()) + self.assertEqual(forwarded, ["SIGTERM"]) + self.assertEqual(records[-1]["child_returncode"], 0) + + def test_leader_exit_cleans_up_surviving_grandchild(self) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "grandchild=os.fork();" + "\nif grandchild == 0:\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n')\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pids" + audit_path = root / "audit.jsonl" + self._write_procfs_fixture(root) + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--grace-seconds", + "0.2", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + wrapper.kill() + wrapper.wait(timeout=5) + self.fail("leader process did not write child PIDs") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_file.read_text( + encoding="utf-8" + ).split() + ) + try: + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual(wrapper.returncode, 0) + self.assertEqual(records[-1]["classification"], "child_exit") + self.assertEqual(records[-1]["child_returncode"], 0) + self.assertEqual(forwarded, ["SIGTERM", "SIGKILL"]) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + def test_configuration_rejects_non_finite_timing(self) -> None: config = watchdog.WatchdogConfig( command=("fake-command",), @@ -335,14 +510,7 @@ def test_configuration_rejects_non_finite_timing(self) -> None: def test_cli_fixture_launches_command_and_propagates_exit(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - (root / "meminfo").write_text( - "MemTotal: 131072 kB\nMemAvailable: 65536 kB\n", - encoding="utf-8", - ) - (root / "swaps").write_text( - "Filename Type Size Used Priority\n", - encoding="utf-8", - ) + self._write_procfs_fixture(root) result = subprocess.run( [ sys.executable, From 93aff41b19864eb89b1cecff358fd50857440d31 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 08:45:06 -0700 Subject: [PATCH 04/56] scripts : classify soft descendant escalation as timeout Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 1 + scripts/strix_memory_watchdog.py | 22 +++++- tests/test_strix_memory_watchdog.py | 108 ++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 5a6558f0a9c0..ccd04f7ab667 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -12,6 +12,7 @@ The wrapper performs these checks and actions: - It calculates used memory as `MemTotal - MemAvailable`. Linux reports these fields in KiB, so the wrapper multiplies each value by 1024 and keeps all accounting as integer bytes. - It sends `SIGTERM` to the process group at 116 GiB used. - It sends `SIGKILL` at 118 GiB used or 30 seconds after `SIGTERM`. +- It reports `grace_timeout` if any descendant requires `SIGKILL` after the soft-threshold grace period, even when the direct child exited earlier. - It sends `SIGKILL` and fails if swap appears or required procfs data becomes unavailable during execution. - It forwards wrapper `SIGINT` or `SIGTERM` to the process group, waits the configured grace period, then sends `SIGKILL` if any group member remains. - It checks the process group after the direct child exits and cleans up remaining descendants before returning the child's classification. diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index 79d7a0013e77..00a9fab78d81 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -385,8 +385,10 @@ def _graceful_cleanup( monotonic: Callable[[], float], sleeper: Callable[[float], None], process_group_status: str = "active", + escalation_result: tuple[str, int, str] | None = None, error: str | None = None, ) -> int: + escalated = False try: if graceful_signal is not None: process_group_status = signal_group( @@ -412,9 +414,15 @@ def _graceful_cleanup( sleeper(min(0.05, deadline - monotonic())) child.poll() if group_alive(child.pid): + escalated = True process_group_status = signal_group( child.pid, signal.SIGKILL ) + signal_reason = ( + escalation_result[2] + if escalation_result is not None + else reason + ) audit.emit( "process_group_signal", **_state_fields( @@ -423,7 +431,7 @@ def _graceful_cleanup( child, child.poll(), process_group_status, - reason, + signal_reason, ), signal="SIGKILL", ) @@ -459,6 +467,8 @@ def _graceful_cleanup( str(exc), ) + if escalated and escalation_result is not None: + classification, exit_code, reason = escalation_result return _emit_final( audit, classification, @@ -528,6 +538,16 @@ def _monitor_child( monotonic, sleeper, group_status, + ( + ( + "grace_timeout", + EXIT_GRACE_TIMEOUT, + "soft-threshold grace period expired with " + "process group members still running", + ) + if soft_stop + else None + ), ) return _emit_final( audit, diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index 2e59eec27512..898298efb9b9 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -499,6 +499,114 @@ def test_leader_exit_cleans_up_surviving_grandchild(self) -> None: time.sleep(0.01) self.assertFalse(self._process_is_running(process_id)) + def test_soft_limit_descendant_escalation_is_grace_timeout(self) -> None: + child_code = ( + "import os,signal,sys,time\n" + "def stop(_signal,_frame):\n" + " raise SystemExit(0)\n" + "signal.signal(signal.SIGTERM,stop)\n" + "grandchild=os.fork()\n" + "if grandchild == 0:\n" + " signal.signal(signal.SIGTERM,signal.SIG_IGN)\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n')\n" + " time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pids" + audit_path = root / "audit.jsonl" + (root / "meminfo").write_text( + "MemTotal: 3145728 kB\n" + "MemAvailable: 2621440 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--soft-gib", + "1", + "--emergency-gib", + "2", + "--grace-seconds", + "0.2", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + wrapper.kill() + wrapper.wait(timeout=5) + self.fail("soft-limit process group did not start") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_file.read_text( + encoding="utf-8" + ).split() + ) + next_meminfo = root / "meminfo.next" + next_meminfo.write_text( + "MemTotal: 3145728 kB\n" + "MemAvailable: 1572864 kB\n", + encoding="utf-8", + ) + next_meminfo.replace(root / "meminfo") + try: + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + final = records[-1] + self.assertEqual(wrapper.returncode, watchdog.EXIT_GRACE_TIMEOUT) + self.assertEqual(final["classification"], "grace_timeout") + self.assertEqual(final["child_returncode"], 0) + self.assertEqual(forwarded, ["SIGTERM", "SIGKILL"]) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + def test_configuration_rejects_non_finite_timing(self) -> None: config = watchdog.WatchdogConfig( command=("fake-command",), From 58533e74ea4cbdd7407db53bb35f047a93d7171b Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 20:27:11 -0700 Subject: [PATCH 05/56] deepseek41 : add cross-runtime trace harness Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/CMakeLists.txt | 11 + tests/test-deepseek41-trace.py | 310 ++++++++++++ tools/CMakeLists.txt | 1 + tools/deepseek-v41-trace/CMakeLists.txt | 8 + tools/deepseek-v41-trace/README.md | 55 +++ tools/deepseek-v41-trace/llama-trace.cpp | 579 ++++++++++++++++++++++ tools/deepseek-v41-trace/preflight.py | 162 +++++++ tools/deepseek-v41-trace/run_ds4.py | 112 +++++ tools/deepseek-v41-trace/run_llama.py | 96 ++++ tools/deepseek-v41-trace/run_matrix.py | 149 ++++++ tools/deepseek-v41-trace/trace_format.py | 584 +++++++++++++++++++++++ 11 files changed, 2067 insertions(+) create mode 100644 tests/test-deepseek41-trace.py create mode 100644 tools/deepseek-v41-trace/CMakeLists.txt create mode 100644 tools/deepseek-v41-trace/README.md create mode 100644 tools/deepseek-v41-trace/llama-trace.cpp create mode 100644 tools/deepseek-v41-trace/preflight.py create mode 100644 tools/deepseek-v41-trace/run_ds4.py create mode 100644 tools/deepseek-v41-trace/run_llama.py create mode 100644 tools/deepseek-v41-trace/run_matrix.py create mode 100644 tools/deepseek-v41-trace/trace_format.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 966e16a7c6c7..cdea2dfc9d79 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -203,6 +203,17 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-engram.cpp) llama_build_and_test(test-llama-archs.cpp) + find_package(Python3 COMPONENTS Interpreter) + if (Python3_Interpreter_FOUND) + llama_test_cmd( + ${Python3_EXECUTABLE} + NAME test-deepseek41-trace + LABEL main + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test-deepseek41-trace.py + ) + endif() + set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") file(MAKE_DIRECTORY "${MODEL_DIR}") diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py new file mode 100644 index 000000000000..3981c45609b2 --- /dev/null +++ b/tests/test-deepseek41-trace.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +import importlib.util +import json +import struct +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).parents[1] / "tools" / "deepseek-v41-trace" / "trace_format.py" +SPEC = importlib.util.spec_from_file_location("dsv41_trace_format", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +trace = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(trace) + + +def manifest(runtime: str = "test") -> dict: + return { + "runtime": runtime, + "revision": "a" * 40, + "build": "test-build", + "model": {"sha256": "1" * 64, "byte_count": 123}, + "prompt": {"sha256": "2" * 64, "byte_count": 3}, + "config": { + "context": 32768, + "decode_steps": 1, + "batch": 512, + "ubatch": 128, + "kv_type_k": "f16", + "kv_type_v": "f16", + "flash_attention": True, + "expert_cache_slots": 8, + "expert_cache_bytes": 4096, + }, + "comparison": {"logits": "byte-identical-f32"}, + "expected": { + "prompt_tokens": 2, + "decode_steps": 1, + "components": { + "prompt.tokens": {"layers": None, "input": "tokens"}, + "engram.row_ids": {"layers": [1], "prefill": "tokens", "decode": "steps"}, + "expert.ids": {"layers": [0], "prefill": "tokens", "decode": "steps"}, + "expert.weights": {"layers": [0], "prefill": "tokens", "decode": "steps"}, + "attn.source": {"layers": [20], "prefill": "tokens", "decode": "steps"}, + "attn.candidate_blocks": {"layers": [20], "prefill": "tokens", "decode": "steps"}, + "attn.candidates": {"layers": [24], "prefill": "tokens", "decode": "steps"}, + "logits.prefill": {"layers": None, "prefill": "final"}, + "logits.decode": {"layers": None, "decode": "steps"}, + "decode.greedy_token": {"layers": None, "decode": "steps"}, + }, + }, + "environment": {}, + "audits": { + "memory": "memory.json", + "swap": "swap.json", + "watchdog": "watchdog.json", + }, + } + + +def add_required_events(writer: object, logits: bytes | None = None) -> None: + writer.add_event( + component="prompt.tokens", + phase="input", + step=0, + token_start=0, + token_count=2, + layer=None, + dtype="i32", + shape=[2], + data=struct.pack(" None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + bits = (0x3F800000, 0x80000000, 0x7FC12345, 0) + data = struct.pack(" None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + with trace.TraceBundleWriter(left, manifest("llama.cpp")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("ds4")) as writer: + add_required_events(writer) + right_bundle = trace.TraceBundle(right) + expert = next(item for item in right_bundle.events if item["component"] == "expert.ids") + expert_blob = right / expert["blob"] + expert_blob.write_bytes(struct.pack( + " None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + writer = trace.TraceBundleWriter(root, manifest()) + with self.assertRaisesRegex(trace.TraceError, "original expert IDs"): + writer.add_event( + component="expert.ids", phase="prefill", step=0, token_start=0, token_count=1, + layer=0, dtype="i32", shape=[1], data=struct.pack(" None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + events_path = root / trace.EVENTS_NAME + events_path.write_bytes(events_path.read_bytes()[:-1]) + with self.assertRaisesRegex(trace.TraceError, "truncated"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + bundle = trace.TraceBundle(root) + event = bundle.events[0] + (root / event["blob"]).write_bytes(b"bad") + with self.assertRaisesRegex(trace.TraceError, "truncated blob|corrupt blob"): + trace.TraceBundle(root) + + def test_report_generation_passes_identical_bundles(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + with trace.TraceBundleWriter(left, manifest("llama.cpp")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("ds4")) as writer: + add_required_events(writer) + result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) + self.assertEqual(result["status"], "TARGET PASS") + self.assertEqual(result["events_compared"], 16) + self.assertIsNone(result["first_divergence"]) + + def test_manifest_mismatch_is_classified(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + left_manifest = manifest("llama.cpp") + right_manifest = manifest("ds4") + right_manifest["prompt"]["sha256"] = "3" * 64 + with trace.TraceBundleWriter(left, left_manifest) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, right_manifest) as writer: + add_required_events(writer) + result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) + self.assertEqual(result["first_divergence"]["classification"], "prompt_identity") + + def test_identically_incomplete_decode_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + incomplete = manifest() + incomplete["config"]["decode_steps"] = 2 + incomplete["expected"]["decode_steps"] = 2 + with trace.TraceBundleWriter(root, incomplete) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "decode step coverage"): + trace.TraceBundle(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index c8ad1db43623..1cf2d8720271 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -38,6 +38,7 @@ else() add_subdirectory(export-lora) endif() add_subdirectory(fit-params) + add_subdirectory(deepseek-v41-trace) if (GGML_METAL) add_subdirectory(tuning) endif() diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt new file mode 100644 index 000000000000..4dc699e3312f --- /dev/null +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -0,0 +1,8 @@ +set(TARGET llama-deepseek-v41-trace) +add_executable(${TARGET} llama-trace.cpp) +target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +if(LLAMA_TOOLS_INSTALL) + install(TARGETS ${TARGET} RUNTIME) +endif() diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md new file mode 100644 index 000000000000..2656aee91066 --- /dev/null +++ b/tools/deepseek-v41-trace/README.md @@ -0,0 +1,55 @@ +# DeepSeek V4.1 correctness traces + +This directory defines the versioned cross-runtime trace format used by issue #48. It compares the unchanged published GGUF between llama.cpp and ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. + +Each trace is a directory: + +- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, and memory/swap/watchdog audit references. +- `events.jsonl` is an ordered stream of content-addressed event records. +- `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. + +The required hard-failure event components are `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. + +Validate or compare bundles: + +```sh +python3 tools/deepseek-v41-trace/trace_format.py validate TRACE +python3 tools/deepseek-v41-trace/trace_format.py compare DS4_TRACE LLAMA_TRACE --report report.json +``` + +The first mismatch is reported by phase, decode step, token range, layer, component, byte offset, and logical element index. All required components use exact byte comparison. There is no tolerance mode. + +## Corpus matrix + +Use only the target model and these repository files: + +```text +tests/corpus/correctness-prose.txt +tests/corpus/correctness-code.txt +tests/corpus/correctness-structured.txt +tests/corpus/correctness-numeric.txt +``` + +Start at context 32768. Repeat a corpus deterministically when more prompt tokens are required, and save the exact repeated bytes before either runtime executes. Run prefill plus at least eight greedy decode steps in one reused context. Boundary runs must vary the prefill chunk size without changing prompt bytes or model settings. + +## Strix execution gate + +`run_ds4.py` verifies the pinned ds4 checkout and refuses model execution when swap is enabled, the watchdog PID file is missing/stale, another matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. + +```sh +python3 tools/deepseek-v41-trace/run_ds4.py \ + --model /mnt/models/deepseek-v41/DeepSeek-V4.1-Flash-Q2.gguf \ + --prompt /path/on/nvme/correctness-prose-32768.txt \ + --output /path/on/nvme/traces/ds4-prose-32768 \ + --watchdog-pid-file /run/user/$(id -u)/dsv41-watchdog.pid \ + --exporter /path/to/pinned-ds4-trace-exporter \ + --exporter-sha256 +``` + +The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision. + +The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-f` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. + +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and writes separate audit files next to the trace directory. + +`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, records their hashes, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Its default context matrix is 32768. Pass `--contexts 32768 65536 98304 131072` only after the 32K target passes. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp new file mode 100644 index 000000000000..185a0407ff13 --- /dev/null +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -0,0 +1,579 @@ +#include "arg.h" +#include "build-info.h" +#include "common.h" +#include "ggml-backend.h" +#include "ggml.h" +extern "C" { +#include "hash/sha256/sha256.h" +} +#include "llama.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using json = nlohmann::ordered_json; + +static constexpr int TRACE_VERSION = 1; +static constexpr const char * TRACE_PREFIX = "dsv41.trace."; + +static std::string sha256_hex(const unsigned char digest[SHA256_DIGEST_SIZE]) { + std::ostringstream stream; + stream << std::hex << std::setfill('0'); + for (size_t i = 0; i < SHA256_DIGEST_SIZE; ++i) { + stream << std::setw(2) << static_cast(digest[i]); + } + return stream.str(); +} + +static std::string sha256_data(const void * data, size_t size) { + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_hash(digest, static_cast(data), size); + return sha256_hex(digest); +} + +static std::string sha256_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open for SHA-256: " + path.string()); + } + + sha256_t state; + sha256_init(&state); + std::vector buffer(1024 * 1024); + while (input) { + input.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + const std::streamsize count = input.gcount(); + if (count > 0) { + sha256_update(&state, buffer.data(), static_cast(count)); + } + } + if (!input.eof()) { + throw std::runtime_error("failed while hashing: " + path.string()); + } + + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_final(&state, digest); + return sha256_hex(digest); +} + +static std::vector read_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input) { + throw std::runtime_error("cannot open: " + path.string()); + } + const std::streamsize size = input.tellg(); + if (size < 0) { + throw std::runtime_error("cannot determine file size: " + path.string()); + } + std::vector result(static_cast(size)); + input.seekg(0); + if (size != 0 && !input.read(reinterpret_cast(result.data()), size)) { + throw std::runtime_error("cannot read: " + path.string()); + } + return result; +} + +static void require_nvme_path(const fs::path & path, const char * label) { + const fs::path absolute = fs::absolute(path).lexically_normal(); + const std::string value = absolute.string(); + if (value == "/mnt/bigspace" || value.rfind("/mnt/bigspace/", 0) == 0) { + throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); + } +} + +static std::string required_environment(const char * name) { + const char * value = std::getenv(name); + if (value == nullptr || value[0] == '\0') { + throw std::runtime_error(std::string("required environment variable is missing: ") + name); + } + return value; +} + +static json audit_reference(const char * environment_name, const char * expected_kind) { + const fs::path path = required_environment(environment_name); + require_nvme_path(path, "audit"); + const std::vector bytes = read_file(path); + json audit; + try { + audit = json::parse(bytes.begin(), bytes.end()); + } catch (const json::exception & error) { + throw std::runtime_error(std::string("invalid audit JSON: ") + error.what()); + } + if (audit.value("kind", "") != expected_kind) { + throw std::runtime_error(std::string("audit kind mismatch for ") + expected_kind); + } + const int64_t created = audit.value("created_unix", INT64_C(0)); + const int64_t now = static_cast(std::time(nullptr)); + if (created <= 0 || now < created || now - created > 300) { + throw std::runtime_error(std::string("audit is stale: ") + expected_kind); + } + if (std::string(expected_kind) == "swap" && audit["data"].value("enabled", true)) { + throw std::runtime_error("swap audit reports enabled swap"); + } +#if defined(__linux__) + if (std::string(expected_kind) == "watchdog") { + const int64_t pid = audit["data"].value("pid", INT64_C(0)); + if (pid <= 1 || !fs::exists("/proc/" + std::to_string(pid))) { + throw std::runtime_error("watchdog audit process is not running"); + } + } +#endif + return { + {"path", fs::absolute(path).lexically_normal().string()}, + {"sha256", sha256_data(bytes.data(), bytes.size())}, + {"created_unix", created}, + }; +} + +static std::string tensor_dtype(const ggml_tensor * tensor) { + switch (tensor->type) { + case GGML_TYPE_F32: return "f32"; + case GGML_TYPE_BF16: return "bf16"; + case GGML_TYPE_I32: return "i32"; + case GGML_TYPE_I8: return "i8"; + default: throw std::runtime_error( + std::string("unsupported trace tensor type: ") + ggml_type_name(tensor->type)); + } +} + +static std::vector tensor_shape(const ggml_tensor * tensor) { + int rank = GGML_MAX_DIMS; + while (rank > 1 && tensor->ne[rank - 1] == 1) { + --rank; + } + std::vector result; + result.reserve(rank); + for (int i = rank - 1; i >= 0; --i) { + result.push_back(tensor->ne[i]); + } + return result; +} + +class trace_writer { +public: + trace_writer(fs::path root, json manifest) : + root(std::move(root)), + blobs(this->root / "blobs"), + manifest(std::move(manifest)) { + if (fs::exists(this->root) && !fs::is_empty(this->root)) { + throw std::runtime_error("trace output directory is not empty: " + this->root.string()); + } + fs::create_directories(blobs); + events.open(this->root / "events.jsonl", std::ios::binary | std::ios::trunc); + if (!events) { + throw std::runtime_error("cannot create events.jsonl"); + } + this->manifest["trace_format"] = "dsv41-trace"; + this->manifest["trace_version"] = TRACE_VERSION; + } + + void set_execution(std::string phase, int step, int64_t token_start, int64_t token_count) { + this->phase = std::move(phase); + this->step = step; + this->token_start = token_start; + this->token_count = token_count; + } + + void add( + const std::string & component, + int layer, + const std::string & dtype, + const std::vector & shape, + const void * data, + size_t size, + const char * semantic_id_space = nullptr) { + if (error_message.size() != 0) { + return; + } + try { + const std::string digest = sha256_data(data, size); + const fs::path blob = blobs / (digest + ".bin"); + if (!fs::exists(blob)) { + const fs::path temp = blob.string() + ".tmp"; + { + std::ofstream output(temp, std::ios::binary | std::ios::trunc); + if (!output || (size != 0 && !output.write(static_cast(data), size))) { + throw std::runtime_error("cannot write trace blob"); + } + } + fs::rename(temp, blob); + } + + json event = { + {"trace_version", TRACE_VERSION}, + {"component", component}, + {"phase", phase}, + {"step", step}, + {"token_start", token_start}, + {"token_count", token_count}, + {"layer", layer >= 0 ? json(layer) : json(nullptr)}, + {"dtype", dtype}, + {"shape", shape}, + {"byte_order", "little"}, + {"byte_count", size}, + {"sha256", digest}, + {"blob", "blobs/" + digest + ".bin"}, + }; + if (semantic_id_space != nullptr) { + event["semantic_id_space"] = semantic_id_space; + } + events << event.dump() << '\n'; + events.flush(); + if (!events) { + throw std::runtime_error("cannot append trace event"); + } + ++event_count; + } catch (const std::exception & error) { + error_message = error.what(); + } + } + + void add_tensor(const ggml_tensor * tensor) { + const std::string name = tensor->name; + std::string component; + const char * semantic_id_space = nullptr; + if (name.rfind("dsv41.trace.engram.row_ids.l", 0) == 0) { + component = "engram.row_ids"; + } else if (name.rfind("dsv41.trace.expert.ids.l", 0) == 0) { + component = "expert.ids"; + semantic_id_space = "original"; + } else if (name.rfind("dsv41.trace.expert.weights.l", 0) == 0) { + component = "expert.weights"; + } else if (name.rfind("dsv41.trace.attn.source.l", 0) == 0) { + component = "attn.source"; + } else if (name.rfind("dsv41.trace.attn.candidate_blocks.l", 0) == 0) { + component = "attn.candidate_blocks"; + } else if (name.rfind("dsv41.trace.attn.candidates.l", 0) == 0) { + component = "attn.candidates"; + } else { + return; + } + + static const std::regex layer_pattern(R"(\.l([0-9]+)$)"); + std::smatch match; + if (!std::regex_search(name, match, layer_pattern)) { + throw std::runtime_error("trace tensor name has no layer suffix: " + name); + } + const int layer = std::stoi(match[1].str()); + const size_t size = ggml_nbytes(tensor); + buffer.resize(size); + ggml_backend_tensor_get(tensor, buffer.data(), 0, size); + add(component, layer, tensor_dtype(tensor), tensor_shape(tensor), buffer.data(), size, semantic_id_space); + } + + bool has_error() const { + return !error_message.empty(); + } + + const std::string & error() const { + return error_message; + } + + void fail(const std::string & message) { + if (error_message.empty()) { + error_message = message; + } + } + + void finish() { + if (has_error()) { + throw std::runtime_error(error_message); + } + events.close(); + manifest["event_count"] = event_count; + const fs::path output = root / "manifest.json"; + const fs::path temp = output.string() + ".tmp"; + { + std::ofstream stream(temp, std::ios::binary | std::ios::trunc); + stream << manifest.dump() << '\n'; + if (!stream) { + throw std::runtime_error("cannot write trace manifest"); + } + } + fs::rename(temp, output); + } + +private: + fs::path root; + fs::path blobs; + std::ofstream events; + json manifest; + std::vector buffer; + std::string phase = "unknown"; + std::string error_message; + int step = 0; + int64_t token_start = 0; + int64_t token_count = 0; + uint64_t event_count = 0; +}; + +static bool trace_callback(ggml_tensor * tensor, bool ask, void * user_data) { + auto * writer = static_cast(user_data); + if (ask) { + return std::string(tensor->name).rfind(TRACE_PREFIX, 0) == 0; + } + try { + writer->add_tensor(tensor); + return !writer->has_error(); + } catch (const std::exception & error) { + writer->fail(error.what()); + std::fprintf(stderr, "trace callback failed: %s\n", error.what()); + return false; + } +} + +static std::vector copy_logits(llama_context * ctx, int32_t n_vocab) { + const float * logits = llama_get_logits_ith(ctx, -1); + if (logits == nullptr) { + throw std::runtime_error("runtime did not produce final-token logits"); + } + return std::vector(logits, logits + n_vocab); +} + +static int32_t greedy_token(const std::vector & logits) { + if (logits.empty()) { + throw std::runtime_error("cannot select from empty logits"); + } + return static_cast(std::max_element(logits.begin(), logits.end()) - logits.begin()); +} + +static void decode_tokens( + llama_context * ctx, + trace_writer & writer, + const std::vector & tokens, + int32_t n_batch) { + int64_t offset = 0; + while (offset < static_cast(tokens.size())) { + const int32_t count = static_cast(std::min(n_batch, tokens.size() - offset)); + llama_batch batch = llama_batch_init(count, 0, 1); + for (int32_t i = 0; i < count; ++i) { + const bool logits = offset + i + 1 == static_cast(tokens.size()); + common_batch_add(batch, tokens[offset + i], offset + i, {0}, logits); + } + writer.set_execution("prefill", 0, offset, count); + const int result = llama_decode(ctx, batch); + llama_batch_free(batch); + if (result != 0) { + throw std::runtime_error("prefill failed at token " + std::to_string(offset)); + } + if (writer.has_error()) { + throw std::runtime_error(writer.error()); + } + offset += count; + } +} + +static std::string model_architecture(const llama_model * model) { + std::array buffer = {}; + const int32_t count = llama_model_meta_val_str( + model, "general.architecture", buffer.data(), buffer.size()); + if (count < 0) { + throw std::runtime_error("model has no general.architecture metadata"); + } + return buffer.data(); +} + +static std::vector command_line(int argc, char ** argv) { + std::vector result; + result.reserve(argc); + for (int i = 0; i < argc; ++i) { + result.emplace_back(argv[i]); + } + return result; +} + +int main(int argc, char ** argv) { + std::setlocale(LC_NUMERIC, "C"); + try { + const uint16_t endian = 1; + if (*reinterpret_cast(&endian) != 1) { + throw std::runtime_error("trace writer requires a little-endian host"); + } + + common_params params; + params.escape = false; + params.warmup = false; + params.ctx_shift = false; + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_RESULTS)) { + return 1; + } + if (params.model.path.empty() || params.prompt_file.empty() || params.out_file.empty()) { + throw std::runtime_error("-m, -f, and -o are required"); + } + if (params.n_predict < 1) { + throw std::runtime_error("-n must request at least one deterministic decode step"); + } + + require_nvme_path(params.model.path, "model"); + require_nvme_path(params.prompt_file, "prompt"); + require_nvme_path(params.out_file, "trace output"); + const json memory_audit = audit_reference("DSV41_TRACE_MEMORY_AUDIT", "memory"); + const json swap_audit = audit_reference("DSV41_TRACE_SWAP_AUDIT", "swap"); + const json watchdog_audit = audit_reference("DSV41_TRACE_WATCHDOG_AUDIT", "watchdog"); + + const std::vector prompt_bytes = read_file(params.prompt_file); + if (params.prompt.size() != prompt_bytes.size() || + !std::equal(prompt_bytes.begin(), prompt_bytes.end(), params.prompt.begin())) { + throw std::runtime_error("parsed prompt differs from exact prompt file bytes"); + } + + const fs::path model_path = fs::absolute(params.model.path).lexically_normal(); + const fs::path prompt_path = fs::absolute(params.prompt_file).lexically_normal(); + const fs::path output_path = fs::absolute(params.out_file).lexically_normal(); + llama_backend_init(); + llama_numa_init(params.numa); + common_init_result_ptr init = common_init_from_params(params); + llama_model * model = init->model(); + llama_context * ctx = init->context(); + if (model == nullptr || ctx == nullptr) { + throw std::runtime_error("failed to initialize llama.cpp"); + } + if (model_architecture(model) != "deepseek41") { + throw std::runtime_error("trace tool requires general.architecture=deepseek41"); + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool add_bos = llama_vocab_get_add_bos(vocab); + const std::vector tokens = common_tokenize(ctx, params.prompt, add_bos, true); + if (tokens.empty()) { + throw std::runtime_error("prompt tokenization produced no tokens"); + } + if (tokens.size() > llama_n_ctx(ctx)) { + throw std::runtime_error("prompt token count exceeds the configured context"); + } + + std::vector all_layers(40); + for (int32_t layer = 0; layer < 40; ++layer) { + all_layers[layer] = layer; + } + + json manifest = { + {"runtime", "llama.cpp"}, + {"revision", llama_commit()}, + {"build", { + {"number", llama_build_number()}, + {"info", llama_build_info()}, + {"compiler", llama_compiler()}, + {"target", llama_build_target()}, + }}, + {"model", { + {"path", model_path.string()}, + {"byte_count", fs::file_size(model_path)}, + {"sha256", sha256_file(model_path)}, + }}, + {"prompt", { + {"path", prompt_path.string()}, + {"byte_count", prompt_bytes.size()}, + {"sha256", sha256_data(prompt_bytes.data(), prompt_bytes.size())}, + }}, + {"config", { + {"context", llama_n_ctx(ctx)}, + {"batch", params.n_batch}, + {"ubatch", params.n_ubatch}, + {"decode_steps", params.n_predict}, + {"kv_type_k", ggml_type_name(params.cache_type_k)}, + {"kv_type_v", ggml_type_name(params.cache_type_v)}, + {"flash_attention", static_cast(params.flash_attn_type)}, + {"gpu_layers", params.n_gpu_layers}, + {"load_mode", static_cast(params.load_mode)}, + {"expert_cache_slots", params.expert_cache_slots}, + {"expert_cache_bytes", static_cast(params.expert_cache_mib) << 20}, + {"tokenizer_add_bos", add_bos}, + {"tokenizer_parse_special", true}, + }}, + {"comparison", { + {"tokens", "exact"}, + {"engram_rows", "exact"}, + {"expert_ids", "exact-original-id-space"}, + {"expert_weights", "byte-identical-f32"}, + {"attention_candidates", "exact"}, + {"logits", "byte-identical-f32"}, + }}, + {"environment", { + {"system_info", common_params_get_system_info(params)}, + {"command", command_line(argc, argv)}, + }}, + {"audits", { + {"memory", memory_audit}, + {"swap", swap_audit}, + {"watchdog", watchdog_audit}, + }}, + {"expected", { + {"prompt_tokens", tokens.size()}, + {"decode_steps", params.n_predict}, + {"components", { + {"prompt.tokens", {{"layers", nullptr}, {"input", "tokens"}}}, + {"engram.row_ids", {{"layers", {1, 14}}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"expert.ids", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"expert.weights", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"attn.source", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"attn.candidate_blocks", {{"layers", {20}}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"attn.candidates", {{"layers", {24, 28, 32, 36}}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"logits.prefill", {{"layers", nullptr}, {"prefill", "final"}}}, + {"logits.decode", {{"layers", nullptr}, {"decode", "steps"}}}, + {"decode.greedy_token", {{"layers", nullptr}, {"decode", "steps"}}}, + }}, + }}, + }; + + trace_writer writer(output_path, std::move(manifest)); + llama_set_eval_callback(ctx, trace_callback, &writer); + + writer.set_execution("input", 0, 0, tokens.size()); + writer.add("prompt.bytes", -1, "bytes", {static_cast(prompt_bytes.size())}, + prompt_bytes.data(), prompt_bytes.size()); + writer.add("prompt.tokens", -1, "i32", {static_cast(tokens.size())}, + tokens.data(), tokens.size()*sizeof(tokens[0])); + + decode_tokens(ctx, writer, tokens, params.n_batch); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); + std::vector logits = copy_logits(ctx, n_vocab); + writer.set_execution("prefill", 0, tokens.size() - 1, 1); + writer.add("logits.prefill", -1, "f32", {n_vocab}, logits.data(), logits.size()*sizeof(float)); + + int64_t position = tokens.size(); + for (int32_t step = 0; step < params.n_predict; ++step) { + const llama_token token = greedy_token(logits); + writer.set_execution("decode", step, position, 1); + writer.add("decode.greedy_token", -1, "i32", {1}, &token, sizeof(token)); + + llama_batch batch = llama_batch_init(1, 0, 1); + common_batch_add(batch, token, position, {0}, true); + const int result = llama_decode(ctx, batch); + llama_batch_free(batch); + if (result != 0) { + throw std::runtime_error("decode failed at step " + std::to_string(step)); + } + if (writer.has_error()) { + throw std::runtime_error(writer.error()); + } + logits = copy_logits(ctx, n_vocab); + writer.add("logits.decode", -1, "f32", {n_vocab}, logits.data(), logits.size()*sizeof(float)); + ++position; + } + + writer.finish(); + llama_backend_free(); + return 0; + } catch (const std::exception & error) { + std::fprintf(stderr, "llama-deepseek-v41-trace: %s\n", error.what()); + return 1; + } +} diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py new file mode 100644 index 000000000000..ca7544d16087 --- /dev/null +++ b/tools/deepseek-v41-trace/preflight.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +import json +import os +import time +from pathlib import Path + +FORBIDDEN_ROOT = Path("/mnt/bigspace") +SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 + + +class PreflightError(RuntimeError): + pass + + +def resolved(path: Path) -> Path: + return path.expanduser().resolve() + + +def require_nvme_path(path: Path, label: str) -> Path: + path = resolved(path) + try: + path.relative_to(FORBIDDEN_ROOT) + except ValueError: + return path + raise PreflightError(f"{label} must not use rotational storage: {path}") + + +def read_proc_lines(path: Path) -> list[str]: + try: + return path.read_text(encoding="ascii").splitlines() + except OSError as error: + raise PreflightError(f"cannot read {path}: {error}") from error + + +def swap_audit() -> dict[str, object]: + lines = read_proc_lines(Path("/proc/swaps")) + entries = [] + for line in lines[1:]: + fields = line.split() + if len(fields) >= 5: + entries.append({ + "path": fields[0], + "type": fields[1], + "size_kib": int(fields[2]), + "used_kib": int(fields[3]), + "priority": int(fields[4]), + }) + return {"enabled": bool(entries), "entries": entries} + + +def memory_audit() -> dict[str, int]: + values: dict[str, int] = {} + for line in read_proc_lines(Path("/proc/meminfo")): + key, value = line.split(":", 1) + fields = value.split() + if fields: + values[key] = int(fields[0]) * 1024 + required = ("MemTotal", "MemAvailable") + if any(key not in values for key in required): + raise PreflightError("/proc/meminfo lacks MemTotal or MemAvailable") + result = { + "mem_total_bytes": values["MemTotal"], + "mem_available_bytes": values["MemAvailable"], + "mem_used_bytes": values["MemTotal"] - values["MemAvailable"], + } + if result["mem_used_bytes"] >= SOFT_MEMORY_LIMIT: + raise PreflightError( + f"host memory use is at or above the 116 GiB soft limit: {result['mem_used_bytes']}") + return result + + +def watchdog_audit(pid_file: Path) -> dict[str, object]: + pid_file = resolved(pid_file) + try: + pid = int(pid_file.read_text(encoding="ascii").strip()) + except (OSError, ValueError) as error: + raise PreflightError(f"watchdog pid file is invalid: {error}") from error + if pid <= 1 or not Path(f"/proc/{pid}").exists(): + raise PreflightError(f"watchdog process {pid} is not running") + try: + command = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip() + except OSError as error: + raise PreflightError(f"cannot inspect watchdog process {pid}: {error}") from error + if not command: + raise PreflightError(f"watchdog process {pid} has no command line") + return {"pid": pid, "pid_file": str(pid_file), "command": command} + + +def matching_workloads(patterns: list[str]) -> list[dict[str, object]]: + matches = [] + excluded = {os.getpid(), os.getppid()} + lowered = [pattern.lower() for pattern in patterns if pattern] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + if pid in excluded: + continue + try: + command = (entry / "cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip() + except OSError: + continue + command_lower = command.lower() + if any(pattern in command_lower for pattern in lowered): + matches.append({"pid": pid, "command": command}) + return matches + + +def run_preflight( + *, + model: Path, + prompt: Path, + output: Path, + watchdog_pid_file: Path, + busy_patterns: list[str], +) -> dict[str, object]: + model = require_nvme_path(model, "model") + prompt = require_nvme_path(prompt, "prompt") + output = require_nvme_path(output, "trace output") + if not model.is_file(): + raise PreflightError(f"model is not a file: {model}") + if not prompt.is_file(): + raise PreflightError(f"prompt is not a file: {prompt}") + swap = swap_audit() + if swap["enabled"]: + raise PreflightError("swap is enabled; model execution is blocked") + watchdog = watchdog_audit(watchdog_pid_file) + workloads = matching_workloads(busy_patterns) + if workloads: + raise PreflightError("active model workload detected: " + json.dumps(workloads, ensure_ascii=True)) + return { + "created_unix": int(time.time()), + "model": str(model), + "prompt": str(prompt), + "output": str(output), + "memory": memory_audit(), + "swap": swap, + "watchdog": watchdog, + "active_workloads": [], + } + + +def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: + root = resolved(root) + if root.exists() and any(root.iterdir()): + raise PreflightError(f"audit directory is not empty: {root}") + root.mkdir(parents=True, exist_ok=True) + result = {} + for key in ("memory", "swap", "watchdog"): + path = root / f"{key}.json" + value = { + "created_unix": audit["created_unix"], + "kind": key, + "data": audit[key], + } + path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + result[key] = str(path) + summary = root / "preflight.json" + summary.write_text(json.dumps(audit, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + result["preflight"] = str(summary) + return result diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py new file mode 100644 index 000000000000..0467c7bc77d5 --- /dev/null +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from preflight import PreflightError, resolved, run_preflight, write_audits +from trace_format import TraceBundle, sha256_file + +DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" + + +def read_revision(checkout: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + text=True, + ).strip() + + +def preflight(args: argparse.Namespace) -> dict[str, object]: + checkout = resolved(args.checkout) + revision = read_revision(checkout) + if revision != DS4_REVISION: + raise PreflightError(f"ds4 revision mismatch: expected {DS4_REVISION}, found {revision}") + result = run_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + watchdog_pid_file=args.watchdog_pid_file, + busy_patterns=args.busy_pattern, + ) + result.update({ + "runtime": "ds4", + "ds4_revision": revision, + "checkout": str(checkout), + "config": { + "context": args.context, + "decode_steps": args.decode_steps, + "prefill_chunk": args.prefill_chunk, + }, + }) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed launcher for the pinned ds4 trace exporter") + parser.add_argument("--checkout", type=Path, default=Path("/home/papa/src/ds4-v41")) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--prompt", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--watchdog-pid-file", type=Path, required=True) + parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) + parser.add_argument("--exporter", type=Path, required=True) + parser.add_argument("--exporter-sha256", required=True) + parser.add_argument("--context", type=int, default=32768) + parser.add_argument("--decode-steps", type=int, default=8) + parser.add_argument("--prefill-chunk", type=int, default=512) + parser.add_argument("--preflight-only", action="store_true") + args = parser.parse_args() + + try: + audit = preflight(args) + if args.preflight_only: + print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) + return 0 + + exporter = resolved(args.exporter) + if not exporter.is_file() or not os.access(exporter, os.X_OK): + raise PreflightError(f"trace exporter is not executable: {exporter}") + exporter_sha256 = sha256_file(exporter) + if exporter_sha256 != args.exporter_sha256: + raise PreflightError( + f"trace exporter SHA-256 mismatch: expected {args.exporter_sha256}, found {exporter_sha256}") + audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} + output = resolved(args.output) + if output.exists() and any(output.iterdir()): + raise PreflightError(f"trace output directory is not empty: {output}") + audits = write_audits(Path(str(output) + ".audit"), audit) + command = [ + str(exporter), + "--model", str(resolved(args.model)), + "--prompt-file", str(resolved(args.prompt)), + "--output", str(output), + "--context", str(args.context), + "--decode-steps", str(args.decode_steps), + "--prefill-chunk", str(args.prefill_chunk), + "--memory-audit", audits["memory"], + "--swap-audit", audits["swap"], + "--watchdog-audit", audits["watchdog"], + ] + print("exec:", shlex.join(command), file=sys.stderr) + result = subprocess.run(command, cwd=resolved(args.checkout), check=False) + if result.returncode != 0: + return result.returncode + bundle = TraceBundle(output) + if bundle.manifest.get("runtime") != "ds4": + raise PreflightError("ds4 exporter wrote a non-ds4 trace") + if bundle.manifest.get("revision") != DS4_REVISION: + raise PreflightError( + f"ds4 trace revision mismatch: expected {DS4_REVISION}, found {bundle.manifest.get('revision')}") + return 0 + except PreflightError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py new file mode 100644 index 000000000000..81244dd23933 --- /dev/null +++ b/tools/deepseek-v41-trace/run_llama.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from preflight import PreflightError, resolved, run_preflight, write_audits +from trace_format import TraceBundle + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed launcher for llama.cpp DeepSeek V4.1 traces") + parser.add_argument("--exporter", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--prompt", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--watchdog-pid-file", type=Path, required=True) + parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) + parser.add_argument("--context", type=int, default=32768) + parser.add_argument("--decode-steps", type=int, default=8) + parser.add_argument("--batch", type=int, default=2048) + parser.add_argument("--ubatch", type=int, default=512) + parser.add_argument("--expert-cache-slots", type=int, required=True) + parser.add_argument("--expert-cache-mib", type=int, required=True) + parser.add_argument("--gpu-layers", type=int, default=99) + parser.add_argument("--preflight-only", action="store_true") + args = parser.parse_args() + + try: + audit = run_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + watchdog_pid_file=args.watchdog_pid_file, + busy_patterns=args.busy_pattern, + ) + audit["runtime"] = "llama.cpp" + audit["config"] = { + "context": args.context, + "decode_steps": args.decode_steps, + "batch": args.batch, + "ubatch": args.ubatch, + "expert_cache_slots": args.expert_cache_slots, + "expert_cache_mib": args.expert_cache_mib, + "gpu_layers": args.gpu_layers, + } + if args.preflight_only: + print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) + return 0 + + exporter = resolved(args.exporter) + if not exporter.is_file() or not os.access(exporter, os.X_OK): + raise PreflightError(f"trace exporter is not executable: {exporter}") + output = resolved(args.output) + if output.exists() and any(output.iterdir()): + raise PreflightError(f"trace output directory is not empty: {output}") + audits = write_audits(Path(str(output) + ".audit"), audit) + environment = os.environ.copy() + environment["DSV41_TRACE_MEMORY_AUDIT"] = audits["memory"] + environment["DSV41_TRACE_SWAP_AUDIT"] = audits["swap"] + environment["DSV41_TRACE_WATCHDOG_AUDIT"] = audits["watchdog"] + command = [ + str(exporter), + "-m", str(resolved(args.model)), + "-f", str(resolved(args.prompt)), + "-o", str(output), + "-c", str(args.context), + "-n", str(args.decode_steps), + "-b", str(args.batch), + "-ub", str(args.ubatch), + "-ngl", str(args.gpu_layers), + "-fa", "on", + "-ctk", "f16", + "-ctv", "f16", + "--expert-cache-slots", str(args.expert_cache_slots), + "--expert-cache-mib", str(args.expert_cache_mib), + ] + print("exec:", shlex.join(command), file=sys.stderr) + result = subprocess.run(command, env=environment, check=False) + if result.returncode != 0: + return result.returncode + bundle = TraceBundle(output) + if bundle.manifest.get("runtime") != "llama.cpp": + raise PreflightError("llama exporter wrote a non-llama.cpp trace") + return 0 + except PreflightError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py new file mode 100644 index 000000000000..8588840232f7 --- /dev/null +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + +from preflight import PreflightError, require_nvme_path, resolved +from trace_format import TraceBundle, report + +CORPORA = ( + "correctness-prose.txt", + "correctness-code.txt", + "correctness-structured.txt", + "correctness-numeric.txt", +) + + +def run(command: list[str]) -> None: + print("exec:", " ".join(command), file=sys.stderr) + result = subprocess.run(command, check=False) + if result.returncode != 0: + raise RuntimeError(f"command failed with status {result.returncode}") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run the DeepSeek V4.1 cross-runtime corpus matrix") + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--watchdog-pid-file", type=Path, required=True) + parser.add_argument("--llama-runner", type=Path, required=True) + parser.add_argument("--llama-exporter", type=Path, required=True) + parser.add_argument("--ds4-runner", type=Path, required=True) + parser.add_argument("--ds4-exporter", type=Path, required=True) + parser.add_argument("--ds4-exporter-sha256", required=True) + parser.add_argument("--ds4-checkout", type=Path, default=Path("/home/papa/src/ds4-v41")) + parser.add_argument("--contexts", type=int, nargs="+", default=[32768]) + parser.add_argument("--ubatches", type=int, nargs="+", default=[512]) + parser.add_argument("--decode-steps", type=int, default=8) + parser.add_argument("--batch", type=int, default=2048) + parser.add_argument("--expert-cache-slots", type=int, required=True) + parser.add_argument("--expert-cache-mib", type=int, required=True) + args = parser.parse_args() + + try: + repo = resolved(args.repo) + output = require_nvme_path(args.output, "matrix output") + model = require_nvme_path(args.model, "model") + if output.exists() and any(output.iterdir()): + raise PreflightError(f"matrix output directory is not empty: {output}") + inputs = output / "inputs" + inputs.mkdir(parents=True, exist_ok=True) + corpus_records = [] + for name in CORPORA: + source = repo / "tests" / "corpus" / name + if not source.is_file(): + raise PreflightError(f"repository corpus is missing: {source}") + destination = inputs / name + shutil.copyfile(source, destination) + corpus_records.append({ + "name": name, + "source": str(source), + "path": str(destination), + "byte_count": destination.stat().st_size, + "sha256": sha256_file(destination), + }) + + results = [] + for context in args.contexts: + if context < 32768 or context > 131072: + raise PreflightError(f"context is outside the supported 32768..131072 matrix: {context}") + for ubatch in args.ubatches: + for corpus in corpus_records: + stem = Path(corpus["name"]).stem + case = f"{stem}-c{context}-ub{ubatch}" + llama_output = output / "llama" / case + ds4_output = output / "ds4" / case + common = [ + "--model", str(model), + "--prompt", corpus["path"], + "--watchdog-pid-file", str(resolved(args.watchdog_pid_file)), + "--context", str(context), + "--decode-steps", str(args.decode_steps), + ] + run([ + sys.executable, + str(resolved(args.ds4_runner)), + "--checkout", str(resolved(args.ds4_checkout)), + "--exporter", str(resolved(args.ds4_exporter)), + "--exporter-sha256", args.ds4_exporter_sha256, + "--output", str(ds4_output), + "--prefill-chunk", str(ubatch), + *common, + ]) + run([ + sys.executable, + str(resolved(args.llama_runner)), + "--exporter", str(resolved(args.llama_exporter)), + "--output", str(llama_output), + "--batch", str(args.batch), + "--ubatch", str(ubatch), + "--expert-cache-slots", str(args.expert_cache_slots), + "--expert-cache-mib", str(args.expert_cache_mib), + *common, + ]) + comparison = report(TraceBundle(ds4_output), TraceBundle(llama_output)) + result_path = output / "reports" / f"{case}.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text( + json.dumps(comparison, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + results.append({"case": case, **comparison}) + if comparison["status"] != "TARGET PASS": + raise RuntimeError(f"correctness mismatch in {case}: {comparison['first_divergence']}") + + summary = { + "status": "TARGET PASS", + "model": str(model), + "corpora": corpus_records, + "contexts": args.contexts, + "ubatches": args.ubatches, + "decode_steps": args.decode_steps, + "cases": results, + } + (output / "summary.json").write_text( + json.dumps(summary, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + return 0 + except (PreflightError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py new file mode 100644 index 000000000000..24045b727e09 --- /dev/null +++ b/tools/deepseek-v41-trace/trace_format.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import struct +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, BinaryIO, Iterable + +TRACE_FORMAT = "dsv41-trace" +TRACE_VERSION = 1 +MANIFEST_NAME = "manifest.json" +EVENTS_NAME = "events.jsonl" +BLOBS_DIR = "blobs" + +DTYPE_SIZES = { + "f32": 4, + "bf16": 2, + "i32": 4, + "u32": 4, + "i8": 1, + "u8": 1, + "bytes": 1, +} + +HARD_FAILURE_COMPONENTS = ( + "prompt.tokens", + "engram.row_ids", + "expert.ids", + "expert.weights", + "attn.source", + "attn.candidate_blocks", + "attn.candidates", + "logits.prefill", + "logits.decode", + "decode.greedy_token", +) + + +class TraceError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Mismatch: + classification: str + component: str + phase: str + step: int + token_start: int + layer: int | None + detail: str + element_index: int | None = None + byte_offset: int | None = None + + def as_dict(self) -> dict[str, Any]: + result = { + "classification": self.classification, + "component": self.component, + "phase": self.phase, + "step": self.step, + "token_start": self.token_start, + "layer": self.layer, + "detail": self.detail, + } + if self.element_index is not None: + result["element_index"] = self.element_index + if self.byte_offset is not None: + result["byte_offset"] = self.byte_offset + return result + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(data: Any) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def element_count(shape: Iterable[int]) -> int: + count = 1 + for dim in shape: + if not isinstance(dim, int) or dim < 0: + raise TraceError(f"invalid shape dimension: {dim!r}") + count *= dim + return count + + +def expected_bytes(event: dict[str, Any]) -> int: + dtype = event.get("dtype") + if dtype not in DTYPE_SIZES: + raise TraceError(f"unsupported dtype: {dtype!r}") + shape = event.get("shape") + if not isinstance(shape, list): + raise TraceError("event shape must be an array") + return element_count(shape) * DTYPE_SIZES[dtype] + + +def event_key(event: dict[str, Any]) -> tuple[Any, ...]: + return ( + event.get("phase", ""), + int(event.get("step", -1)), + int(event.get("token_start", -1)), + event.get("layer"), + event.get("component", ""), + ) + + +def event_order(key: tuple[Any, ...]) -> tuple[Any, ...]: + phase_order = {"input": 0, "prefill": 1, "decode": 2} + component_order = { + "prompt.bytes": 0, + "prompt.tokens": 1, + "engram.row_ids": 2, + "expert.ids": 3, + "expert.weights": 4, + "attn.source": 5, + "attn.candidate_blocks": 6, + "attn.candidates": 7, + "decode.greedy_token": 8, + "logits.prefill": 9, + "logits.decode": 10, + } + return ( + phase_order.get(key[0], 99), + key[2], + key[1], + -1 if key[3] is None else key[3], + component_order.get(key[4], 99), + key[4], + ) + + +def classify(component: str) -> str: + if component == "prompt.tokens": + return "tokenizer" + if component == "engram.row_ids": + return "engram_row" + if component == "expert.ids": + return "routing_original_expert" + if component == "expert.weights": + return "routing_weight" + if component.startswith("attn.candidate"): + return "attention_candidate" + if component == "attn.source": + return "attention_source" + if component == "logits.prefill": + return "prefill_logits" + if component == "logits.decode": + return "decode_logits" + if component == "decode.greedy_token": + return "decode_token" + return "trace_data" + + +def validate_event(event: dict[str, Any]) -> None: + required = { + "trace_version", + "component", + "phase", + "step", + "token_start", + "token_count", + "layer", + "dtype", + "shape", + "byte_order", + "byte_count", + "sha256", + "blob", + } + missing = sorted(required - event.keys()) + if missing: + raise TraceError(f"event is missing fields: {', '.join(missing)}") + if event["trace_version"] != TRACE_VERSION: + raise TraceError(f"unsupported event version: {event['trace_version']!r}") + if event["byte_order"] != "little": + raise TraceError("trace blobs must use little-endian byte order") + if event["byte_count"] != expected_bytes(event): + raise TraceError("event byte_count does not match dtype and shape") + digest = event["sha256"] + if not isinstance(digest, str) or len(digest) != 64: + raise TraceError("event sha256 is invalid") + if event["blob"] != f"{BLOBS_DIR}/{digest}.bin": + raise TraceError("event blob path is not content addressed") + component = event["component"] + if component == "expert.ids": + if event.get("semantic_id_space") != "original": + raise TraceError("expert.ids must contain original expert IDs, not cache slot IDs") + if "slot" in component or event.get("semantic_id_space") == "cache_slot": + raise TraceError("cache slot IDs are forbidden in correctness traces") + + +class TraceBundleWriter: + def __init__(self, root: Path, manifest: dict[str, Any]): + self.root = root + self.blobs = root / BLOBS_DIR + if root.exists() and any(root.iterdir()): + raise TraceError(f"trace output directory is not empty: {root}") + self.blobs.mkdir(parents=True, exist_ok=True) + self.events = (root / EVENTS_NAME).open("x", encoding="ascii", newline="\n") + self.manifest = dict(manifest) + self.manifest["trace_format"] = TRACE_FORMAT + self.manifest["trace_version"] = TRACE_VERSION + self.event_count = 0 + + def add_event( + self, + *, + component: str, + phase: str, + step: int, + token_start: int, + token_count: int, + layer: int | None, + dtype: str, + shape: list[int], + data: bytes, + semantic_id_space: str | None = None, + ) -> dict[str, Any]: + digest = sha256_bytes(data) + blob_rel = f"{BLOBS_DIR}/{digest}.bin" + event = { + "trace_version": TRACE_VERSION, + "component": component, + "phase": phase, + "step": step, + "token_start": token_start, + "token_count": token_count, + "layer": layer, + "dtype": dtype, + "shape": shape, + "byte_order": "little", + "byte_count": len(data), + "sha256": digest, + "blob": blob_rel, + } + if semantic_id_space is not None: + event["semantic_id_space"] = semantic_id_space + validate_event(event) + blob_path = self.root / blob_rel + if not blob_path.exists(): + temp = blob_path.with_suffix(".tmp") + temp.write_bytes(data) + os.replace(temp, blob_path) + self.events.write(canonical_json(event) + "\n") + self.events.flush() + self.event_count += 1 + return event + + def close(self) -> None: + if self.events.closed: + return + self.events.close() + self.manifest["event_count"] = self.event_count + manifest_path = self.root / MANIFEST_NAME + temp = manifest_path.with_suffix(".tmp") + temp.write_text(canonical_json(self.manifest) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + def __enter__(self) -> "TraceBundleWriter": + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + self.events.close() + if exc_type is None: + self.manifest["event_count"] = self.event_count + manifest_path = self.root / MANIFEST_NAME + temp = manifest_path.with_suffix(".tmp") + temp.write_text(canonical_json(self.manifest) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +class TraceBundle: + def __init__(self, root: Path, verify_blobs: bool = True): + self.root = root + try: + self.manifest = json.loads((root / MANIFEST_NAME).read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise TraceError(f"cannot read manifest: {error}") from error + if self.manifest.get("trace_format") != TRACE_FORMAT: + raise TraceError("manifest trace_format mismatch") + if self.manifest.get("trace_version") != TRACE_VERSION: + raise TraceError("manifest trace_version mismatch") + self.events = self._read_events(verify_blobs) + if self.manifest.get("event_count") != len(self.events): + raise TraceError("manifest event_count mismatch") + self._validate_coverage() + + def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: + result = [] + try: + stream: BinaryIO + with (self.root / EVENTS_NAME).open("rb") as stream: + for line_number, raw in enumerate(stream, 1): + if not raw.endswith(b"\n"): + raise TraceError(f"events.jsonl is truncated at line {line_number}") + try: + event = json.loads(raw.decode("ascii")) + except (UnicodeError, json.JSONDecodeError) as error: + raise TraceError(f"invalid event at line {line_number}: {error}") from error + validate_event(event) + if verify_blobs: + data = self.read_blob(event) + if len(data) != event["byte_count"]: + raise TraceError(f"truncated blob for event line {line_number}") + if sha256_bytes(data) != event["sha256"]: + raise TraceError(f"corrupt blob for event line {line_number}") + result.append(event) + except OSError as error: + raise TraceError(f"cannot read events: {error}") from error + return result + + def read_blob(self, event: dict[str, Any]) -> bytes: + try: + return (self.root / event["blob"]).read_bytes() + except OSError as error: + raise TraceError(f"cannot read blob {event['blob']}: {error}") from error + + def _validate_coverage(self) -> None: + expected = self.manifest.get("expected") + if not isinstance(expected, dict): + raise TraceError("manifest expected coverage is missing") + prompt_tokens = expected.get("prompt_tokens") + decode_steps = expected.get("decode_steps") + components = expected.get("components") + if not isinstance(prompt_tokens, int) or prompt_tokens <= 0: + raise TraceError("expected prompt_tokens is invalid") + if not isinstance(decode_steps, int) or decode_steps <= 0: + raise TraceError("expected decode_steps is invalid") + if self.manifest.get("config", {}).get("decode_steps") != decode_steps: + raise TraceError("expected decode_steps does not match config") + if not isinstance(components, dict): + raise TraceError("expected components are invalid") + + by_component: dict[str, list[dict[str, Any]]] = {} + for event in self.events: + by_component.setdefault(event["component"], []).append(event) + for component in HARD_FAILURE_COMPONENTS: + if component not in components: + raise TraceError(f"expected coverage lacks {component}") + rules = components[component] + events = by_component.get(component, []) + if not events: + raise TraceError(f"trace lacks required component: {component}") + expected_layers = rules.get("layers") + if expected_layers is not None: + actual_layers = sorted({event["layer"] for event in events}) + if actual_layers != expected_layers: + raise TraceError( + f"{component} layer coverage mismatch: expected {expected_layers}, found {actual_layers}") + input_coverage = rules.get("input") + prefill_coverage = rules.get("prefill") + decode_coverage = rules.get("decode") + if input_coverage == "tokens": + input_events = [event for event in events if event["phase"] == "input"] + if len(input_events) != 1 or input_events[0]["token_start"] != 0 or input_events[0]["token_count"] != prompt_tokens: + raise TraceError(f"{component} input coverage is incomplete") + for layer in expected_layers or [None]: + layer_events = [event for event in events if event["layer"] == layer] + if prefill_coverage == "tokens": + prefill = sorted( + (event for event in layer_events if event["phase"] == "prefill"), + key=lambda event: event["token_start"], + ) + frontier = 0 + for event in prefill: + if event["token_start"] != frontier or event["token_count"] <= 0: + raise TraceError(f"{component} prefill coverage has a gap or overlap at token {frontier}") + frontier += event["token_count"] + if frontier != prompt_tokens: + raise TraceError( + f"{component} prefill coverage ends at {frontier}, expected {prompt_tokens}") + elif prefill_coverage == "final": + prefill = [event for event in layer_events if event["phase"] == "prefill"] + if len(prefill) != 1 or prefill[0]["token_start"] != prompt_tokens - 1 or prefill[0]["token_count"] != 1: + raise TraceError(f"{component} final prefill coverage is invalid") + if decode_coverage == "steps": + decode = sorted( + (event for event in layer_events if event["phase"] == "decode"), + key=lambda event: event["step"], + ) + if [event["step"] for event in decode] != list(range(decode_steps)): + raise TraceError(f"{component} decode step coverage is incomplete") + for event in decode: + if event["token_start"] != prompt_tokens + event["step"] or event["token_count"] != 1: + raise TraceError(f"{component} decode token coordinates are invalid") + + +def first_byte_difference(left: bytes, right: bytes) -> int | None: + for index, (a, b) in enumerate(zip(left, right)): + if a != b: + return index + if len(left) != len(right): + return min(len(left), len(right)) + return None + + +def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: + checks = ( + ("model.sha256", "model_identity"), + ("prompt.sha256", "prompt_identity"), + ("prompt.byte_count", "prompt_identity"), + ("config.context", "configuration"), + ("config.decode_steps", "configuration"), + ("comparison.logits", "comparison_policy"), + ) + for dotted, classification in checks: + left_value: Any = left.manifest + right_value: Any = right.manifest + for key in dotted.split("."): + left_value = left_value.get(key) if isinstance(left_value, dict) else None + right_value = right_value.get(key) if isinstance(right_value, dict) else None + if left_value != right_value: + return Mismatch( + classification, + "manifest", + "metadata", + -1, + -1, + None, + f"{dotted} differs: {left_value!r} != {right_value!r}", + ) + return None + + +def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: + mismatch = compare_manifests(left, right) + if mismatch is not None: + return mismatch + + for bundle_name, bundle in (("left", left), ("right", right)): + components = {event["component"] for event in bundle.events} + missing = sorted(set(HARD_FAILURE_COMPONENTS) - components) + if missing: + return Mismatch( + "artifact_missing", + "manifest", + "metadata", + -1, + -1, + None, + f"{bundle_name} trace is missing required components: {', '.join(missing)}", + ) + + left_map = {event_key(event): event for event in left.events} + right_map = {event_key(event): event for event in right.events} + if len(left_map) != len(left.events) or len(right_map) != len(right.events): + return Mismatch( + "artifact_duplicate", + "events", + "metadata", + -1, + -1, + None, + "a trace contains duplicate phase/step/token/layer/component coordinates", + ) + all_keys = sorted(set(left_map) | set(right_map), key=event_order) + for key in all_keys: + left_event = left_map.get(key) + right_event = right_map.get(key) + template = left_event or right_event + assert template is not None + if left_event is None or right_event is None: + return Mismatch( + "artifact_missing", + template["component"], + template["phase"], + template["step"], + template["token_start"], + template["layer"], + "event is missing from " + ("left" if left_event is None else "right") + " trace", + ) + for field in ("dtype", "shape", "token_count", "semantic_id_space"): + if left_event.get(field) != right_event.get(field): + return Mismatch( + "artifact_shape", + template["component"], + template["phase"], + template["step"], + template["token_start"], + template["layer"], + f"{field} differs: {left_event.get(field)!r} != {right_event.get(field)!r}", + ) + if left_event["sha256"] == right_event["sha256"]: + continue + left_data = left.read_blob(left_event) + right_data = right.read_blob(right_event) + byte_offset = first_byte_difference(left_data, right_data) + assert byte_offset is not None + item_size = DTYPE_SIZES[left_event["dtype"]] + detail = f"first byte mismatch at {byte_offset}" + item_offset = byte_offset - byte_offset % item_size + if item_offset + item_size <= min(len(left_data), len(right_data)): + left_item = left_data[item_offset:item_offset + item_size] + right_item = right_data[item_offset:item_offset + item_size] + detail += f"; left=0x{left_item.hex()} right=0x{right_item.hex()}" + return Mismatch( + classify(template["component"]), + template["component"], + template["phase"], + template["step"], + template["token_start"], + template["layer"], + detail, + element_index=byte_offset // item_size, + byte_offset=byte_offset, + ) + return None + + +def report(left: TraceBundle, right: TraceBundle) -> dict[str, Any]: + mismatch = compare_bundles(left, right) + if mismatch is None: + return { + "status": "TARGET PASS", + "trace_version": TRACE_VERSION, + "left_runtime": left.manifest.get("runtime"), + "right_runtime": right.manifest.get("runtime"), + "events_compared": len(left.events), + "first_divergence": None, + } + return { + "status": "FAIL", + "trace_version": TRACE_VERSION, + "left_runtime": left.manifest.get("runtime"), + "right_runtime": right.manifest.get("runtime"), + "events_compared": 0, + "first_divergence": mismatch.as_dict(), + } + + +def command_validate(args: argparse.Namespace) -> int: + bundle = TraceBundle(args.bundle) + print(canonical_json({ + "status": "valid", + "runtime": bundle.manifest.get("runtime"), + "events": len(bundle.events), + })) + return 0 + + +def command_compare(args: argparse.Namespace) -> int: + result = report(TraceBundle(args.left), TraceBundle(args.right)) + text = canonical_json(result) + "\n" + if args.report: + args.report.write_text(text, encoding="ascii") + sys.stdout.write(text) + return 0 if result["status"] == "TARGET PASS" else 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Validate and compare DeepSeek V4.1 correctness traces") + subparsers = parser.add_subparsers(dest="command", required=True) + validate_parser = subparsers.add_parser("validate") + validate_parser.add_argument("bundle", type=Path) + validate_parser.set_defaults(func=command_validate) + compare_parser = subparsers.add_parser("compare") + compare_parser.add_argument("left", type=Path) + compare_parser.add_argument("right", type=Path) + compare_parser.add_argument("--report", type=Path) + compare_parser.set_defaults(func=command_compare) + return parser + + +def main() -> int: + args = build_parser().parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) From 7a8d939420104aa49c8aa1102970039963764f16 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 20:28:25 -0700 Subject: [PATCH 06/56] deepseek41 : harden trace validation Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tools/deepseek-v41-trace/llama-trace.cpp | 16 +++++++++++++++- tools/deepseek-v41-trace/trace_format.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 185a0407ff13..42a40f45769c 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -135,11 +135,15 @@ static json audit_reference(const char * environment_name, const char * expected } } #endif - return { + json result = { {"path", fs::absolute(path).lexically_normal().string()}, {"sha256", sha256_data(bytes.data(), bytes.size())}, {"created_unix", created}, }; + if (std::string(expected_kind) == "watchdog") { + result["pid"] = audit["data"].value("pid", INT64_C(0)); + } + return result; } static std::string tensor_dtype(const ggml_tensor * tensor) { @@ -458,6 +462,9 @@ int main(int argc, char ** argv) { if (tokens.size() > llama_n_ctx(ctx)) { throw std::runtime_error("prompt token count exceeds the configured context"); } + if (tokens.size() + static_cast(params.n_predict) > llama_n_ctx(ctx)) { + throw std::runtime_error("prompt plus decode steps exceed the configured context"); + } std::vector all_layers(40); for (int32_t layer = 0; layer < 40; ++layer) { @@ -475,6 +482,7 @@ int main(int argc, char ** argv) { }}, {"model", { {"path", model_path.string()}, + {"architecture", "deepseek41"}, {"byte_count", fs::file_size(model_path)}, {"sha256", sha256_file(model_path)}, }}, @@ -569,6 +577,12 @@ int main(int argc, char ** argv) { ++position; } +#if defined(__linux__) + const int64_t watchdog_pid = watchdog_audit.value("pid", INT64_C(0)); + if (watchdog_pid <= 1 || !fs::exists("/proc/" + std::to_string(watchdog_pid))) { + throw std::runtime_error("watchdog stopped before trace completion"); + } +#endif writer.finish(); llama_backend_free(); return 0; diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 24045b727e09..7ef533a8a618 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -39,6 +39,15 @@ "decode.greedy_token", ) +DEEPSEEK41_LAYERS = { + "engram.row_ids": [1, 14], + "expert.ids": list(range(40)), + "expert.weights": list(range(40)), + "attn.source": list(range(40)), + "attn.candidate_blocks": [20], + "attn.candidates": [24, 28, 32, 36], +} + class TraceError(RuntimeError): pass @@ -344,6 +353,10 @@ def _validate_coverage(self) -> None: raise TraceError("expected decode_steps does not match config") if not isinstance(components, dict): raise TraceError("expected components are invalid") + if self.manifest.get("model", {}).get("architecture") == "deepseek41": + for component, layers in DEEPSEEK41_LAYERS.items(): + if components.get(component, {}).get("layers") != layers: + raise TraceError(f"DeepSeek V4.1 expected layers are invalid for {component}") by_component: dict[str, list[dict[str, Any]]] = {} for event in self.events: @@ -411,6 +424,7 @@ def first_byte_difference(left: bytes, right: bytes) -> int | None: def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: checks = ( ("model.sha256", "model_identity"), + ("model.architecture", "model_identity"), ("prompt.sha256", "prompt_identity"), ("prompt.byte_count", "prompt_identity"), ("config.context", "configuration"), From 62545e5399cff8347f4beafce85963d93b121a50 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 20:31:08 -0700 Subject: [PATCH 07/56] deepseek41 : attest trace executables Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 17 ++++++++--- tools/deepseek-v41-trace/llama-trace.cpp | 2 ++ tools/deepseek-v41-trace/run_ds4.py | 2 ++ tools/deepseek-v41-trace/run_llama.py | 5 +++- tools/deepseek-v41-trace/trace_format.py | 38 ++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 3981c45609b2..18b593693bb9 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -18,7 +18,7 @@ def manifest(runtime: str = "test") -> dict: return { "runtime": runtime, "revision": "a" * 40, - "build": "test-build", + "build": {"sha256": "3" * 64}, "model": {"sha256": "1" * 64, "byte_count": 123}, "prompt": {"sha256": "2" * 64, "byte_count": 3}, "config": { @@ -51,9 +51,9 @@ def manifest(runtime: str = "test") -> dict: }, "environment": {}, "audits": { - "memory": "memory.json", - "swap": "swap.json", - "watchdog": "watchdog.json", + "memory": {"path": "memory.json", "sha256": "4" * 64, "created_unix": 1}, + "swap": {"path": "swap.json", "sha256": "5" * 64, "created_unix": 1}, + "watchdog": {"path": "watchdog.json", "sha256": "6" * 64, "created_unix": 1}, }, } @@ -267,6 +267,15 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: with self.assertRaisesRegex(trace.TraceError, "truncated blob|corrupt blob"): trace.TraceBundle(root) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + bad_manifest = manifest() + bad_manifest["audits"]["watchdog"] = "watchdog.json" + with trace.TraceBundleWriter(root, bad_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "watchdog audit reference"): + trace.TraceBundle(root) + def test_report_generation_passes_identical_bundles(self) -> None: with tempfile.TemporaryDirectory() as temp: left = Path(temp) / "left" diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 42a40f45769c..1a47391a67c2 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -479,6 +479,8 @@ int main(int argc, char ** argv) { {"info", llama_build_info()}, {"compiler", llama_compiler()}, {"target", llama_build_target()}, + {"path", fs::absolute(argv[0]).lexically_normal().string()}, + {"sha256", sha256_file(fs::absolute(argv[0]).lexically_normal())}, }}, {"model", { {"path", model_path.string()}, diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 0467c7bc77d5..a209669db6d8 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -102,6 +102,8 @@ def main() -> int: if bundle.manifest.get("revision") != DS4_REVISION: raise PreflightError( f"ds4 trace revision mismatch: expected {DS4_REVISION}, found {bundle.manifest.get('revision')}") + if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: + raise PreflightError("ds4 trace build SHA-256 does not match the executed exporter") return 0 except PreflightError as error: print(f"error: {error}", file=sys.stderr) diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 81244dd23933..cd7e1b872af0 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -9,7 +9,7 @@ from pathlib import Path from preflight import PreflightError, resolved, run_preflight, write_audits -from trace_format import TraceBundle +from trace_format import TraceBundle, sha256_file def main() -> int: @@ -55,6 +55,7 @@ def main() -> int: exporter = resolved(args.exporter) if not exporter.is_file() or not os.access(exporter, os.X_OK): raise PreflightError(f"trace exporter is not executable: {exporter}") + exporter_sha256 = sha256_file(exporter) output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") @@ -86,6 +87,8 @@ def main() -> int: bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "llama.cpp": raise PreflightError("llama exporter wrote a non-llama.cpp trace") + if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: + raise PreflightError("llama trace build SHA-256 does not match the executed exporter") return 0 except PreflightError as error: print(f"error: {error}", file=sys.stderr) diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 7ef533a8a618..535f50a5fe2f 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -4,6 +4,7 @@ import hashlib import json import os +import re import struct import sys from dataclasses import dataclass @@ -303,6 +304,7 @@ def __init__(self, root: Path, verify_blobs: bool = True): raise TraceError("manifest trace_format mismatch") if self.manifest.get("trace_version") != TRACE_VERSION: raise TraceError("manifest trace_version mismatch") + self._validate_manifest() self.events = self._read_events(verify_blobs) if self.manifest.get("event_count") != len(self.events): raise TraceError("manifest event_count mismatch") @@ -332,6 +334,42 @@ def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: raise TraceError(f"cannot read events: {error}") from error return result + def _validate_manifest(self) -> None: + for key in ("runtime", "revision", "build", "model", "prompt", "config", "comparison", "environment", "audits"): + if key not in self.manifest: + raise TraceError(f"manifest is missing {key}") + if not isinstance(self.manifest["runtime"], str) or not self.manifest["runtime"]: + raise TraceError("manifest runtime is invalid") + if not isinstance(self.manifest["revision"], str) or not self.manifest["revision"]: + raise TraceError("manifest revision is invalid") + if not isinstance(self.manifest["build"], dict): + raise TraceError("manifest build is invalid") + if re.fullmatch(r"[0-9a-f]{64}", self.manifest["build"].get("sha256", "")) is None: + raise TraceError("manifest build SHA-256 is invalid") + for section in ("model", "prompt"): + if not isinstance(self.manifest[section], dict): + raise TraceError(f"manifest {section} is invalid") + digest = self.manifest[section].get("sha256") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError(f"manifest {section} SHA-256 is invalid") + if not isinstance(self.manifest[section].get("byte_count"), int): + raise TraceError(f"manifest {section} byte_count is invalid") + for section in ("config", "comparison", "environment", "audits"): + if not isinstance(self.manifest[section], dict): + raise TraceError(f"manifest {section} is invalid") + if self.manifest["comparison"].get("logits") != "byte-identical-f32": + raise TraceError("logit comparison policy must be byte-identical-f32") + for kind in ("memory", "swap", "watchdog"): + audit = self.manifest["audits"].get(kind) + if not isinstance(audit, dict): + raise TraceError(f"manifest {kind} audit reference is invalid") + if not isinstance(audit.get("path"), str) or not audit["path"]: + raise TraceError(f"manifest {kind} audit path is invalid") + if re.fullmatch(r"[0-9a-f]{64}", audit.get("sha256", "")) is None: + raise TraceError(f"manifest {kind} audit SHA-256 is invalid") + if not isinstance(audit.get("created_unix"), int) or audit["created_unix"] <= 0: + raise TraceError(f"manifest {kind} audit timestamp is invalid") + def read_blob(self, event: dict[str, Any]) -> bytes: try: return (self.root / event["blob"]).read_bytes() From 520706ea3a733a95bd5376f40a86bec27c84629c Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 20:56:44 -0700 Subject: [PATCH 08/56] deepseek41 : harden trace audit contract Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 321 +++++++++++++++++--- tools/deepseek-v41-trace/CMakeLists.txt | 7 +- tools/deepseek-v41-trace/README.md | 33 +- tools/deepseek-v41-trace/llama-trace.cpp | 91 +++++- tools/deepseek-v41-trace/preflight.py | 124 +++++++- tools/deepseek-v41-trace/prompt-builder.cpp | 131 ++++++++ tools/deepseek-v41-trace/run_ds4.py | 30 +- tools/deepseek-v41-trace/run_llama.py | 131 ++++++-- tools/deepseek-v41-trace/run_matrix.py | 115 ++++++- tools/deepseek-v41-trace/trace_format.py | 223 +++++++++++++- 10 files changed, 1083 insertions(+), 123 deletions(-) create mode 100644 tools/deepseek-v41-trace/prompt-builder.cpp diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 18b593693bb9..13c8a1fb2811 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -3,24 +3,65 @@ import importlib.util import json import struct +import sys import tempfile import unittest +from argparse import Namespace from pathlib import Path -MODULE_PATH = Path(__file__).parents[1] / "tools" / "deepseek-v41-trace" / "trace_format.py" +TRACE_DIR = Path(__file__).parents[1] / "tools" / "deepseek-v41-trace" +sys.path.insert(0, str(TRACE_DIR)) +MODULE_PATH = TRACE_DIR / "trace_format.py" SPEC = importlib.util.spec_from_file_location("dsv41_trace_format", MODULE_PATH) assert SPEC is not None and SPEC.loader is not None trace = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(trace) +import run_llama +import preflight -def manifest(runtime: str = "test") -> dict: - return { +AUDIT_RECORDS = { + "memory": { + "created_unix": 1, + "kind": "memory", + "data": {"mem_total_bytes": 128, "mem_available_bytes": 64, "mem_used_bytes": 64}, + }, + "swap": { + "created_unix": 1, + "kind": "swap", + "data": {"enabled": False, "entries": []}, + }, + "watchdog": { + "created_unix": 1, + "kind": "watchdog", + "data": { + "pid": 123, + "start_time_ticks": 456, + "command_sha256": "7" * 64, + "heartbeat_path": "/run/user/123/watchdog.heartbeat", + "heartbeat_unix": 1, + "max_heartbeat_age_seconds": 30, + }, + }, +} + + +def audit_bytes(kind: str) -> bytes: + return (json.dumps(AUDIT_RECORDS[kind], sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + +def manifest(runtime: str = "llama.cpp") -> dict: + result = { "runtime": runtime, - "revision": "a" * 40, + "revision": trace.DS4_REVISION if runtime == "ds4" else "a" * 40, "build": {"sha256": "3" * 64}, - "model": {"sha256": "1" * 64, "byte_count": 123}, - "prompt": {"sha256": "2" * 64, "byte_count": 3}, + "model": {"sha256": trace.MODEL_SHA256, "byte_count": 123, "architecture": "deepseek41"}, + "prompt": { + "sha256": trace.sha256_bytes(b"abc"), + "byte_count": 3, + "corpus_name": "correctness-prose.txt", + "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + }, "config": { "context": 32768, "decode_steps": 1, @@ -31,19 +72,33 @@ def manifest(runtime: str = "test") -> dict: "flash_attention": True, "expert_cache_slots": 8, "expert_cache_bytes": 4096, + "deepseek41": { + "layer_count": 40, + "vocab_size": 129280, + "engram_layers": [1, 14], + "engram_rows_per_token": 4, + "expert_count": 384, + "experts_used": 6, + "candidate_source_layer": 20, + "candidate_topk_blocks": 2048, + "candidate_block_size": 8, + "index_top_k": 512, + "candidate_propagation_layers": [24, 28, 32, 36], + }, }, "comparison": {"logits": "byte-identical-f32"}, "expected": { "prompt_tokens": 2, "decode_steps": 1, "components": { + "prompt.bytes": {"layers": None, "input": "tokens"}, "prompt.tokens": {"layers": None, "input": "tokens"}, - "engram.row_ids": {"layers": [1], "prefill": "tokens", "decode": "steps"}, - "expert.ids": {"layers": [0], "prefill": "tokens", "decode": "steps"}, - "expert.weights": {"layers": [0], "prefill": "tokens", "decode": "steps"}, - "attn.source": {"layers": [20], "prefill": "tokens", "decode": "steps"}, + "engram.row_ids": {"layers": [1, 14], "prefill": "tokens", "decode": "steps"}, + "expert.ids": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "expert.weights": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.source": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, "attn.candidate_blocks": {"layers": [20], "prefill": "tokens", "decode": "steps"}, - "attn.candidates": {"layers": [24], "prefill": "tokens", "decode": "steps"}, + "attn.candidates": {"layers": [24, 28, 32, 36], "prefill": "tokens", "decode": "steps"}, "logits.prefill": {"layers": None, "prefill": "final"}, "logits.decode": {"layers": None, "decode": "steps"}, "decode.greedy_token": {"layers": None, "decode": "steps"}, @@ -51,14 +106,42 @@ def manifest(runtime: str = "test") -> dict: }, "environment": {}, "audits": { - "memory": {"path": "memory.json", "sha256": "4" * 64, "created_unix": 1}, - "swap": {"path": "swap.json", "sha256": "5" * 64, "created_unix": 1}, - "watchdog": {"path": "watchdog.json", "sha256": "6" * 64, "created_unix": 1}, + kind: { + "path": f"audits/{trace.sha256_bytes(audit_bytes(kind))}.json", + "sha256": trace.sha256_bytes(audit_bytes(kind)), + "created_unix": 1, + } + for kind in ("memory", "swap", "watchdog") }, } + if runtime == "llama.cpp": + result["candidate"] = { + "repository": trace.REPOSITORY, + "revision": "a" * 40, + "base_revision": "b" * 40, + "diff_sha256": "c" * 64, + "executable_sha256": "3" * 64, + } + return result def add_required_events(writer: object, logits: bytes | None = None) -> None: + audit_root = writer.root / "audits" + audit_root.mkdir(exist_ok=True) + for kind in ("memory", "swap", "watchdog"): + data = audit_bytes(kind) + (audit_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) + writer.add_event( + component="prompt.bytes", + phase="input", + step=0, + token_start=0, + token_count=2, + layer=None, + dtype="bytes", + shape=[3], + data=b"abc", + ) writer.add_event( component="prompt.tokens", phase="input", @@ -77,9 +160,9 @@ def add_required_events(writer: object, logits: bytes | None = None) -> None: token_start=0, token_count=2, layer=1, - dtype="u32", - shape=[2, 4], - data=struct.pack(" None: token_count=2, layer=0, dtype="i32", - shape=[2, 6], + shape=[6, 2], data=struct.pack(" None: token_count=2, layer=0, dtype="f32", - shape=[2, 6], + shape=[6, 2], data=struct.pack(" None: token_count=2, layer=20, dtype="i32", - shape=[2], + shape=[1, 2], data=struct.pack(" None: token_count=2, layer=24, dtype="i32", - shape=[2, 2], - data=struct.pack(" None: token_count=1, layer=None, dtype="f32", - shape=[4], - data=logits if logits is not None else struct.pack(" None: ) writer.add_event( component="engram.row_ids", phase="decode", step=0, token_start=2, token_count=1, - layer=1, dtype="u32", shape=[1, 4], data=struct.pack(" None: token_count=1, layer=None, dtype="f32", - shape=[4], - data=logits if logits is not None else struct.pack(" None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" bits = (0x3F800000, 0x80000000, 0x7FC12345, 0) - data = struct.pack(" None: with tempfile.TemporaryDirectory() as temp: left = Path(temp) / "left" right = Path(temp) / "right" - with trace.TraceBundleWriter(left, manifest("llama.cpp")) as writer: + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: add_required_events(writer) - with trace.TraceBundleWriter(right, manifest("ds4")) as writer: + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: add_required_events(writer) right_bundle = trace.TraceBundle(right) expert = next(item for item in right_bundle.events if item["component"] == "expert.ids") - expert_blob = right / expert["blob"] - expert_blob.write_bytes(struct.pack( - " None: divergence = result["first_divergence"] self.assertEqual(divergence["classification"], "routing_original_expert") self.assertEqual(divergence["layer"], 0) - self.assertEqual(divergence["element_index"], 2) + self.assertEqual(divergence["element_index"], 8) + self.assertEqual(divergence["token_index"], 1) + self.assertEqual(divergence["component_element_index"], 2) + + def test_llama_runner_preserves_binary_prompt_bytes(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + model = root / "model.gguf" + prompt = root / "prompt.txt" + exporter = root / "exporter" + output = root / "trace" + model.write_bytes(b"model") + prompt.write_bytes(b"line with trailing newline\n") + args = Namespace( + model=model, + prompt=prompt, + context=32768, + decode_steps=8, + batch=2048, + ubatch=512, + gpu_layers=99, + expert_cache_slots=8, + expert_cache_mib=4096, + ) + command = run_llama.build_command(args, exporter, output) + self.assertEqual(command[command.index("-bf") + 1], str(prompt.resolve())) + self.assertNotIn("-f", command) def test_rejects_cache_slot_id_space(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -276,30 +424,109 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: with self.assertRaisesRegex(trace.TraceError, "watchdog audit reference"): trace.TraceBundle(root) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest() + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + (root / trace_manifest["audits"]["memory"]["path"]).unlink() + with self.assertRaisesRegex(trace.TraceError, "memory audit evidence"): + trace.TraceBundle(root) + + def test_rejects_unpinned_ds4_revision(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["revision"] = "a" * 40 + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "ds4 revision"): + trace.TraceBundle(root) + + def test_rejects_wrong_component_schema_and_same_bundle_compare(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + events_path = root / trace.EVENTS_NAME + events = [ + json.loads(line) + for line in events_path.read_text(encoding="ascii").splitlines() + ] + expert = next(event for event in events if event["component"] == "expert.ids") + expert["dtype"] = "f32" + events_path.write_text( + "".join(trace.canonical_json(event) + "\n" for event in events), + encoding="ascii", + ) + with self.assertRaisesRegex(trace.TraceError, "expert.ids dtype"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + bundle = trace.TraceBundle(root) + result = trace.report(bundle, bundle) + self.assertEqual(result["first_divergence"]["classification"], "artifact_identity") + + def test_rejects_empty_variable_width_components(self) -> None: + for component in ("attn.source", "attn.candidate_blocks"): + with self.subTest(component=component), tempfile.TemporaryDirectory() as temp: + writer = trace.TraceBundleWriter(Path(temp) / "trace", manifest()) + with self.assertRaisesRegex(trace.TraceError, "nonzero"): + writer.add_event( + component=component, + phase="prefill", + step=0, + token_start=0, + token_count=2, + layer=20, + dtype="i32", + shape=[0, 2], + data=b"", + ) + writer.events.close() + + def test_watchdog_stat_parser_handles_parentheses(self) -> None: + fields = ["S", *[str(value) for value in range(4, 23)]] + self.assertEqual(preflight.proc_start_time_ticks(f"123 (watch) dog) {' '.join(fields)}"), 22) + def test_report_generation_passes_identical_bundles(self) -> None: with tempfile.TemporaryDirectory() as temp: left = Path(temp) / "left" right = Path(temp) / "right" - with trace.TraceBundleWriter(left, manifest("llama.cpp")) as writer: + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: add_required_events(writer) - with trace.TraceBundleWriter(right, manifest("ds4")) as writer: + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: add_required_events(writer) result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) self.assertEqual(result["status"], "TARGET PASS") - self.assertEqual(result["events_compared"], 16) + self.assertEqual(result["events_compared"], 259) self.assertIsNone(result["first_divergence"]) def test_manifest_mismatch_is_classified(self) -> None: with tempfile.TemporaryDirectory() as temp: left = Path(temp) / "left" right = Path(temp) / "right" - left_manifest = manifest("llama.cpp") - right_manifest = manifest("ds4") - right_manifest["prompt"]["sha256"] = "3" * 64 + left_manifest = manifest("ds4") + right_manifest = manifest("llama.cpp") + right_prompt = b"abd" + right_manifest["prompt"]["sha256"] = trace.sha256_bytes(right_prompt) with trace.TraceBundleWriter(left, left_manifest) as writer: add_required_events(writer) with trace.TraceBundleWriter(right, right_manifest) as writer: add_required_events(writer) + events_path = right / trace.EVENTS_NAME + events = [json.loads(line) for line in events_path.read_text(encoding="ascii").splitlines()] + prompt_event = next(event for event in events if event["component"] == "prompt.bytes") + prompt_event["sha256"] = trace.sha256_bytes(right_prompt) + prompt_event["blob"] = f"blobs/{prompt_event['sha256']}.bin" + (right / prompt_event["blob"]).write_bytes(right_prompt) + events_path.write_text( + "".join(trace.canonical_json(event) + "\n" for event in events), + encoding="ascii", + ) result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) self.assertEqual(result["first_divergence"]["classification"], "prompt_identity") diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index 4dc699e3312f..3f311dd4410f 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -3,6 +3,11 @@ add_executable(${TARGET} llama-trace.cpp) target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) +set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) +add_executable(${PROMPT_TARGET} prompt-builder.cpp) +target_link_libraries(${PROMPT_TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${PROMPT_TARGET} PRIVATE cxx_std_17) + if(LLAMA_TOOLS_INSTALL) - install(TARGETS ${TARGET} RUNTIME) + install(TARGETS ${TARGET} ${PROMPT_TARGET} RUNTIME) endif() diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 2656aee91066..ee76ae51590d 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -4,11 +4,14 @@ This directory defines the versioned cross-runtime trace format used by issue #4 Each trace is a directory: -- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, and memory/swap/watchdog audit references. +- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, and content-addressed memory/swap/watchdog audit references. - `events.jsonl` is an ordered stream of content-addressed event records. - `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. +- `audits/.json` stores the immutable safety evidence referenced by the manifest. -The required hard-failure event components are `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. +The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. + +Internal tensors use raw ggml dimension order. The validator requires Engram rows as i32 `[4, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with `token_count` in the second dimension, layer-20 candidate blocks as rank-2 i32 with width at most 2048, propagated candidates as i32 `[512, token_count]`, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. Validate or compare bundles: @@ -17,7 +20,7 @@ python3 tools/deepseek-v41-trace/trace_format.py validate TRACE python3 tools/deepseek-v41-trace/trace_format.py compare DS4_TRACE LLAMA_TRACE --report report.json ``` -The first mismatch is reported by phase, decode step, token range, layer, component, byte offset, and logical element index. All required components use exact byte comparison. There is no tolerance mode. +The first mismatch is reported by phase, decode step, exact token, layer, component, byte offset, flat element index, and per-token component element index. All required components use exact byte comparison. There is no tolerance mode. A ds4 bundle is invalid unless it reports revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. ## Corpus matrix @@ -30,26 +33,38 @@ tests/corpus/correctness-structured.txt tests/corpus/correctness-numeric.txt ``` -Start at context 32768. Repeat a corpus deterministically when more prompt tokens are required, and save the exact repeated bytes before either runtime executes. Run prefill plus at least eight greedy decode steps in one reused context. Boundary runs must vary the prefill chunk size without changing prompt bytes or model settings. +Their SHA-256 values are fixed in `trace_format.py`; the matrix refuses modified corpus bytes. The published GGUF must have SHA-256 `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`. + +Start at context 32768. `llama-deepseek-v41-prompt-builder` loads only the GGUF vocabulary, repeats each repository corpus deterministically, truncates the token sequence to `context - decode_steps`, detokenizes it, and requires exact token round-trip before saving the prompt. `run_matrix.py` creates each prompt once before either runtime executes and reuses its exact bytes for every chunk-boundary case. Run prefill plus at least eight greedy decode steps in one reused context. ## Strix execution gate -`run_ds4.py` verifies the pinned ds4 checkout and refuses model execution when swap is enabled, the watchdog PID file is missing/stale, another matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. +`run_ds4.py` verifies the pinned ds4 checkout and refuses model execution when swap is enabled, the watchdog lease or heartbeat is missing/stale, another matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. + +The watchdog lease is JSON, not a bare PID: + +```json +{"pid":1234,"start_time_ticks":5678,"command_sha256":"","heartbeat_path":"/run/user/1000/dsv41-watchdog.heartbeat","max_heartbeat_age_seconds":30} +``` + +The watchdog must update the heartbeat file with the current Unix timestamp at least every 30 seconds. The wrappers verify the PID, Linux process start time, exact command bytes, and heartbeat before and after execution. The llama exporter repeats the same identity and heartbeat check before finalizing its trace. ```sh python3 tools/deepseek-v41-trace/run_ds4.py \ --model /mnt/models/deepseek-v41/DeepSeek-V4.1-Flash-Q2.gguf \ --prompt /path/on/nvme/correctness-prose-32768.txt \ + --corpus-name correctness-prose.txt \ + --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ --output /path/on/nvme/traces/ds4-prose-32768 \ --watchdog-pid-file /run/user/$(id -u)/dsv41-watchdog.pid \ --exporter /path/to/pinned-ds4-trace-exporter \ --exporter-sha256 ``` -The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision. +The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision and its build SHA-256 matches the executed file. -The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-f` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. +The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and writes separate audit files next to the trace directory. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and embeds content-addressed audit evidence in the trace. It requires the exact candidate revision, full-graph base revision, expected base-to-candidate binary diff SHA-256, and repository path. It rejects a dirty checkout or an exporter whose embedded build revision or executable hash does not match that attestation. -`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, records their hashes, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Its default context matrix is 32768. Pass `--contexts 32768 65536 98304 131072` only after the 32K target passes. +`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, records their hashes, builds exact-length prompt artifacts, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build. Its default context matrix is 32768. Pass `--contexts 32768 65536 98304 131072` only after the 32K target passes. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 1a47391a67c2..925e4c9520f1 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -12,6 +12,7 @@ extern "C" { #include #include +#include #include #include #include @@ -106,6 +107,58 @@ static std::string required_environment(const char * name) { return value; } +#if defined(__linux__) +static uint64_t proc_start_time_ticks(int64_t pid) { + const std::vector bytes = read_file("/proc/" + std::to_string(pid) + "/stat"); + const std::string stat(bytes.begin(), bytes.end()); + const size_t command_end = stat.rfind(')'); + if (command_end == std::string::npos) { + throw std::runtime_error("watchdog process stat is invalid"); + } + std::istringstream fields(stat.substr(command_end + 2)); + std::string value; + for (int field = 3; field <= 22; ++field) { + if (!(fields >> value)) { + throw std::runtime_error("watchdog process stat is truncated"); + } + } + return std::stoull(value); +} + +static void validate_watchdog(const json & data) { + const int64_t pid = data.value("pid", INT64_C(0)); + if (pid <= 1 || !fs::exists("/proc/" + std::to_string(pid))) { + throw std::runtime_error("watchdog process is not running"); + } + if (proc_start_time_ticks(pid) != data.value("start_time_ticks", UINT64_C(0))) { + throw std::runtime_error("watchdog process start time changed"); + } + const std::vector command = read_file("/proc/" + std::to_string(pid) + "/cmdline"); + if (sha256_data(command.data(), command.size()) != data.value("command_sha256", "")) { + throw std::runtime_error("watchdog process command changed"); + } + const fs::path heartbeat_path = data.value("heartbeat_path", ""); + const int64_t max_age = data.value("max_heartbeat_age_seconds", INT64_C(0)); + if (heartbeat_path.empty() || max_age <= 0 || max_age > 30) { + throw std::runtime_error("watchdog heartbeat configuration is invalid"); + } + const std::vector heartbeat_bytes = read_file(heartbeat_path); + const std::string heartbeat_text(heartbeat_bytes.begin(), heartbeat_bytes.end()); + size_t parsed = 0; + const int64_t heartbeat = std::stoll(heartbeat_text, &parsed); + while (parsed < heartbeat_text.size() && std::isspace(static_cast(heartbeat_text[parsed]))) { + ++parsed; + } + if (parsed != heartbeat_text.size()) { + throw std::runtime_error("watchdog heartbeat is invalid"); + } + const int64_t now = static_cast(std::time(nullptr)); + if (heartbeat <= 0 || heartbeat > now || now - heartbeat > max_age) { + throw std::runtime_error("watchdog heartbeat is stale"); + } +} +#endif + static json audit_reference(const char * environment_name, const char * expected_kind) { const fs::path path = required_environment(environment_name); require_nvme_path(path, "audit"); @@ -129,10 +182,7 @@ static json audit_reference(const char * environment_name, const char * expected } #if defined(__linux__) if (std::string(expected_kind) == "watchdog") { - const int64_t pid = audit["data"].value("pid", INT64_C(0)); - if (pid <= 1 || !fs::exists("/proc/" + std::to_string(pid))) { - throw std::runtime_error("watchdog audit process is not running"); - } + validate_watchdog(audit["data"]); } #endif json result = { @@ -141,7 +191,7 @@ static json audit_reference(const char * environment_name, const char * expected {"created_unix", created}, }; if (std::string(expected_kind) == "watchdog") { - result["pid"] = audit["data"].value("pid", INT64_C(0)); + result["data"] = audit["data"]; } return result; } @@ -164,7 +214,7 @@ static std::vector tensor_shape(const ggml_tensor * tensor) { } std::vector result; result.reserve(rank); - for (int i = rank - 1; i >= 0; --i) { + for (int i = 0; i < rank; ++i) { result.push_back(tensor->ne[i]); } return result; @@ -362,10 +412,10 @@ static void decode_tokens( llama_context * ctx, trace_writer & writer, const std::vector & tokens, - int32_t n_batch) { + int32_t n_ubatch) { int64_t offset = 0; while (offset < static_cast(tokens.size())) { - const int32_t count = static_cast(std::min(n_batch, tokens.size() - offset)); + const int32_t count = static_cast(std::min(n_ubatch, tokens.size() - offset)); llama_batch batch = llama_batch_init(count, 0, 1); for (int32_t i = 0; i < count; ++i) { const bool logits = offset + i + 1 == static_cast(tokens.size()); @@ -420,7 +470,7 @@ int main(int argc, char ** argv) { return 1; } if (params.model.path.empty() || params.prompt_file.empty() || params.out_file.empty()) { - throw std::runtime_error("-m, -f, and -o are required"); + throw std::runtime_error("-m, -bf, and -o are required"); } if (params.n_predict < 1) { throw std::runtime_error("-n must request at least one deterministic decode step"); @@ -456,6 +506,7 @@ int main(int argc, char ** argv) { const llama_vocab * vocab = llama_model_get_vocab(model); const bool add_bos = llama_vocab_get_add_bos(vocab); const std::vector tokens = common_tokenize(ctx, params.prompt, add_bos, true); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); if (tokens.empty()) { throw std::runtime_error("prompt tokenization produced no tokens"); } @@ -507,6 +558,19 @@ int main(int argc, char ** argv) { {"expert_cache_bytes", static_cast(params.expert_cache_mib) << 20}, {"tokenizer_add_bos", add_bos}, {"tokenizer_parse_special", true}, + {"deepseek41", { + {"layer_count", 40}, + {"vocab_size", n_vocab}, + {"engram_layers", {1, 14}}, + {"engram_rows_per_token", 4}, + {"expert_count", 384}, + {"experts_used", 6}, + {"candidate_source_layer", 20}, + {"candidate_topk_blocks", 2048}, + {"candidate_block_size", 8}, + {"index_top_k", 512}, + {"candidate_propagation_layers", {24, 28, 32, 36}}, + }}, }}, {"comparison", { {"tokens", "exact"}, @@ -529,6 +593,7 @@ int main(int argc, char ** argv) { {"prompt_tokens", tokens.size()}, {"decode_steps", params.n_predict}, {"components", { + {"prompt.bytes", {{"layers", nullptr}, {"input", "tokens"}}}, {"prompt.tokens", {{"layers", nullptr}, {"input", "tokens"}}}, {"engram.row_ids", {{"layers", {1, 14}}, {"prefill", "tokens"}, {"decode", "steps"}}}, {"expert.ids", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, @@ -552,8 +617,7 @@ int main(int argc, char ** argv) { writer.add("prompt.tokens", -1, "i32", {static_cast(tokens.size())}, tokens.data(), tokens.size()*sizeof(tokens[0])); - decode_tokens(ctx, writer, tokens, params.n_batch); - const int32_t n_vocab = llama_vocab_n_tokens(vocab); + decode_tokens(ctx, writer, tokens, params.n_ubatch); std::vector logits = copy_logits(ctx, n_vocab); writer.set_execution("prefill", 0, tokens.size() - 1, 1); writer.add("logits.prefill", -1, "f32", {n_vocab}, logits.data(), logits.size()*sizeof(float)); @@ -580,10 +644,7 @@ int main(int argc, char ** argv) { } #if defined(__linux__) - const int64_t watchdog_pid = watchdog_audit.value("pid", INT64_C(0)); - if (watchdog_pid <= 1 || !fs::exists("/proc/" + std::to_string(watchdog_pid))) { - throw std::runtime_error("watchdog stopped before trace completion"); - } + validate_watchdog(watchdog_audit["data"]); #endif writer.finish(); llama_backend_free(); diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index ca7544d16087..7649352cbfde 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 import json +import hashlib import os +import re +import shutil import time from pathlib import Path FORBIDDEN_ROOT = Path("/mnt/bigspace") SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 +MAX_WATCHDOG_HEARTBEAT_AGE = 30 class PreflightError(RuntimeError): @@ -70,21 +74,70 @@ def memory_audit() -> dict[str, int]: return result -def watchdog_audit(pid_file: Path) -> dict[str, object]: - pid_file = resolved(pid_file) +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def proc_start_time_ticks(stat: str) -> int: + command_end = stat.rfind(")") + if command_end < 0: + raise PreflightError("watchdog process stat is invalid") + fields = stat[command_end + 2:].split() + if len(fields) < 20: + raise PreflightError("watchdog process stat is truncated") + return int(fields[19]) + + +def read_heartbeat(path: Path, max_age_seconds: int) -> int: try: - pid = int(pid_file.read_text(encoding="ascii").strip()) + heartbeat = int(path.read_text(encoding="ascii").strip()) except (OSError, ValueError) as error: - raise PreflightError(f"watchdog pid file is invalid: {error}") from error + raise PreflightError(f"watchdog heartbeat is invalid: {error}") from error + now = int(time.time()) + if heartbeat <= 0 or heartbeat > now or now - heartbeat > max_age_seconds: + raise PreflightError("watchdog heartbeat is stale") + return heartbeat + + +def watchdog_audit(pid_file: Path) -> dict[str, object]: + pid_file = require_nvme_path(pid_file, "watchdog lease") + try: + lease = json.loads(pid_file.read_text(encoding="ascii")) + pid = int(lease["pid"]) + expected_start = int(lease["start_time_ticks"]) + expected_command_sha256 = str(lease["command_sha256"]) + heartbeat_path = require_nvme_path(Path(lease["heartbeat_path"]), "watchdog heartbeat") + max_age_seconds = int(lease.get("max_heartbeat_age_seconds", MAX_WATCHDOG_HEARTBEAT_AGE)) + except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + raise PreflightError(f"watchdog lease is invalid: {error}") from error + if re.fullmatch(r"[0-9a-f]{64}", expected_command_sha256) is None: + raise PreflightError("watchdog lease command SHA-256 is invalid") + if max_age_seconds <= 0 or max_age_seconds > MAX_WATCHDOG_HEARTBEAT_AGE: + raise PreflightError(f"watchdog heartbeat age must be within 1..{MAX_WATCHDOG_HEARTBEAT_AGE} seconds") if pid <= 1 or not Path(f"/proc/{pid}").exists(): raise PreflightError(f"watchdog process {pid} is not running") try: - command = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip() - except OSError as error: + command_bytes = Path(f"/proc/{pid}/cmdline").read_bytes() + start_time_ticks = proc_start_time_ticks(Path(f"/proc/{pid}/stat").read_text(encoding="ascii")) + except (OSError, ValueError) as error: raise PreflightError(f"cannot inspect watchdog process {pid}: {error}") from error + command_sha256 = sha256_bytes(command_bytes) + if start_time_ticks != expected_start or command_sha256 != expected_command_sha256: + raise PreflightError("watchdog process identity does not match its lease") + heartbeat = read_heartbeat(heartbeat_path, max_age_seconds) + command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() if not command: raise PreflightError(f"watchdog process {pid} has no command line") - return {"pid": pid, "pid_file": str(pid_file), "command": command} + return { + "pid": pid, + "pid_file": str(pid_file), + "start_time_ticks": start_time_ticks, + "command": command, + "command_sha256": command_sha256, + "heartbeat_path": str(heartbeat_path), + "heartbeat_unix": heartbeat, + "max_heartbeat_age_seconds": max_age_seconds, + } def matching_workloads(patterns: list[str]) -> list[dict[str, object]]: @@ -160,3 +213,60 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: summary.write_text(json.dumps(audit, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") result["preflight"] = str(summary) return result + + +def embed_audits(trace_root: Path, audits: dict[str, str]) -> dict[str, dict[str, object]]: + trace_root = resolved(trace_root) + embedded_root = trace_root / "audits" + embedded_root.mkdir(parents=True, exist_ok=True) + result = {} + for kind in ("memory", "swap", "watchdog"): + source = resolved(Path(audits[kind])) + data = source.read_bytes() + digest = sha256_bytes(data) + destination = embedded_root / f"{digest}.json" + if destination.exists() and destination.read_bytes() != data: + raise PreflightError(f"content-addressed audit collision: {destination}") + if not destination.exists(): + shutil.copyfile(source, destination) + try: + record = json.loads(data.decode("ascii")) + created = int(record["created_unix"]) + except (UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot embed {kind} audit: {error}") from error + result[kind] = { + "path": f"audits/{digest}.json", + "sha256": digest, + "created_unix": created, + } + return result + + +def bind_embedded_audits(trace_root: Path, audits: dict[str, str]) -> None: + trace_root = resolved(trace_root) + manifest_path = trace_root / "manifest.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot bind trace audits: {error}") from error + manifest["audits"] = embed_audits(trace_root, audits) + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +def bind_prompt_provenance(trace_root: Path, corpus_name: str, corpus_sha256: str) -> None: + trace_root = resolved(trace_root) + manifest_path = trace_root / "manifest.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot bind prompt provenance: {error}") from error + prompt = manifest.get("prompt") + if not isinstance(prompt, dict): + raise PreflightError("trace manifest prompt is invalid") + prompt["corpus_name"] = corpus_name + prompt["corpus_sha256"] = corpus_sha256 + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) diff --git a/tools/deepseek-v41-trace/prompt-builder.cpp b/tools/deepseek-v41-trace/prompt-builder.cpp new file mode 100644 index 000000000000..82adbcd2d534 --- /dev/null +++ b/tools/deepseek-v41-trace/prompt-builder.cpp @@ -0,0 +1,131 @@ +#include "common.h" +#include "llama.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using json = nlohmann::ordered_json; + +static std::string read_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open: " + path.string()); + } + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +static void require_nvme_path(const fs::path & path, const char * label) { + const std::string value = fs::absolute(path).lexically_normal().string(); + if (value == "/mnt/bigspace" || value.rfind("/mnt/bigspace/", 0) == 0) { + throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); + } +} + +static std::string argument(int argc, char ** argv, const std::string & name) { + for (int index = 1; index + 1 < argc; ++index) { + if (argv[index] == name) { + return argv[index + 1]; + } + } + throw std::runtime_error("missing argument: " + name); +} + +static std::string model_architecture(const llama_model * model) { + char buffer[128] = {}; + if (llama_model_meta_val_str(model, "general.architecture", buffer, sizeof(buffer)) < 0) { + throw std::runtime_error("model has no general.architecture metadata"); + } + return buffer; +} + +int main(int argc, char ** argv) { + try { + const fs::path model_path = argument(argc, argv, "--model"); + const fs::path corpus_path = argument(argc, argv, "--corpus"); + const fs::path output_path = argument(argc, argv, "--output"); + const int64_t target_tokens = std::stoll(argument(argc, argv, "--tokens")); + if (target_tokens < 2) { + throw std::runtime_error("--tokens must be at least 2"); + } + require_nvme_path(model_path, "model"); + require_nvme_path(corpus_path, "corpus"); + require_nvme_path(output_path, "prompt output"); + if (fs::exists(output_path)) { + throw std::runtime_error("prompt output already exists: " + output_path.string()); + } + + const std::string corpus = read_file(corpus_path); + if (corpus.empty()) { + throw std::runtime_error("corpus is empty"); + } + + llama_backend_init(); + llama_model_params model_params = llama_model_default_params(); + model_params.vocab_only = true; + llama_model * model = llama_model_load_from_file(model_path.string().c_str(), model_params); + if (model == nullptr) { + throw std::runtime_error("cannot load model vocabulary"); + } + if (model_architecture(model) != "deepseek41") { + llama_model_free(model); + throw std::runtime_error("prompt builder requires general.architecture=deepseek41"); + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool add_bos = llama_vocab_get_add_bos(vocab); + + std::string repeated = corpus; + std::vector tokens = common_tokenize(vocab, repeated, add_bos, true); + while (tokens.size() < static_cast(target_tokens)) { + if (repeated.size() > (size_t(1) << 31)) { + llama_model_free(model); + throw std::runtime_error("repeated prompt exceeds 2 GiB"); + } + repeated += repeated; + tokens = common_tokenize(vocab, repeated, add_bos, true); + } + tokens.resize(static_cast(target_tokens)); + std::vector content_tokens = tokens; + if (add_bos) { + if (content_tokens.front() != llama_vocab_bos(vocab)) { + llama_model_free(model); + throw std::runtime_error("tokenized prompt does not begin with the configured BOS token"); + } + content_tokens.erase(content_tokens.begin()); + } + const std::string prompt = common_detokenize(vocab, content_tokens, true); + const std::vector verified = common_tokenize(vocab, prompt, add_bos, true); + if (verified != tokens) { + llama_model_free(model); + throw std::runtime_error("constructed prompt does not round-trip to the target token IDs"); + } + + if (!output_path.parent_path().empty()) { + fs::create_directories(output_path.parent_path()); + } + std::ofstream output(output_path, std::ios::binary | std::ios::trunc); + if (!output || !output.write(prompt.data(), static_cast(prompt.size()))) { + llama_model_free(model); + throw std::runtime_error("cannot write prompt output"); + } + output.close(); + std::printf("%s\n", json({ + {"target_tokens", target_tokens}, + {"actual_tokens", verified.size()}, + {"byte_count", prompt.size()}, + {"add_bos", add_bos}, + }).dump().c_str()); + llama_model_free(model); + return 0; + } catch (const std::exception & error) { + std::fprintf(stderr, "error: %s\n", error.what()); + return 1; + } +} diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index a209669db6d8..fcb27030f2e3 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -8,17 +8,21 @@ import sys from pathlib import Path -from preflight import PreflightError, resolved, run_preflight, write_audits -from trace_format import TraceBundle, sha256_file +from preflight import PreflightError, bind_embedded_audits, bind_prompt_provenance, resolved, run_preflight, write_audits +from trace_format import CORPUS_SHA256, MODEL_SHA256, TraceBundle, TraceError, sha256_file DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" def read_revision(checkout: Path) -> str: - return subprocess.check_output( - ["git", "-C", str(checkout), "rev-parse", "HEAD"], - text=True, - ).strip() + try: + return subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + text=True, + stderr=subprocess.STDOUT, + ).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"cannot read ds4 revision: {error}") from error def preflight(args: argparse.Namespace) -> dict[str, object]: @@ -56,6 +60,8 @@ def main() -> int: parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) parser.add_argument("--exporter", type=Path, required=True) parser.add_argument("--exporter-sha256", required=True) + parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) + parser.add_argument("--corpus-sha256", required=True) parser.add_argument("--context", type=int, default=32768) parser.add_argument("--decode-steps", type=int, default=8) parser.add_argument("--prefill-chunk", type=int, default=512) @@ -63,6 +69,8 @@ def main() -> int: args = parser.parse_args() try: + if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: + raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") audit = preflight(args) if args.preflight_only: print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) @@ -75,6 +83,9 @@ def main() -> int: if exporter_sha256 != args.exporter_sha256: raise PreflightError( f"trace exporter SHA-256 mismatch: expected {args.exporter_sha256}, found {exporter_sha256}") + model_sha256 = sha256_file(resolved(args.model)) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} output = resolved(args.output) if output.exists() and any(output.iterdir()): @@ -96,6 +107,9 @@ def main() -> int: result = subprocess.run(command, cwd=resolved(args.checkout), check=False) if result.returncode != 0: return result.returncode + preflight(args) + bind_embedded_audits(output, audits) + bind_prompt_provenance(output, args.corpus_name, args.corpus_sha256) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "ds4": raise PreflightError("ds4 exporter wrote a non-ds4 trace") @@ -104,8 +118,10 @@ def main() -> int: f"ds4 trace revision mismatch: expected {DS4_REVISION}, found {bundle.manifest.get('revision')}") if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: raise PreflightError("ds4 trace build SHA-256 does not match the executed exporter") + if bundle.manifest.get("model", {}).get("sha256") != MODEL_SHA256: + raise PreflightError("ds4 trace model SHA-256 does not match the published GGUF") return 0 - except PreflightError as error: + except (PreflightError, TraceError) as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index cd7e1b872af0..d1af2663b5e0 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import hashlib import json import os import shlex @@ -8,13 +9,102 @@ import sys from pathlib import Path -from preflight import PreflightError, resolved, run_preflight, write_audits -from trace_format import TraceBundle, sha256_file +from preflight import PreflightError, bind_embedded_audits, bind_prompt_provenance, resolved, run_preflight, write_audits +from trace_format import CORPUS_SHA256, MODEL_SHA256, REPOSITORY, TraceBundle, TraceError, sha256_file + + +def git_output(repo: Path, *args: str) -> bytes: + try: + return subprocess.check_output(["git", "-C", str(repo), *args], stderr=subprocess.STDOUT) + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"git {' '.join(args)} failed: {error}") from error + + +def candidate_attestation(args: argparse.Namespace, exporter_sha256: str) -> dict[str, str]: + repo = resolved(args.repo) + revision = git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() + base_revision = git_output(repo, "rev-parse", args.base_revision).decode("ascii").strip() + if revision != args.candidate_revision: + raise PreflightError( + f"candidate revision mismatch: expected {args.candidate_revision}, found {revision}") + if base_revision != args.base_revision: + raise PreflightError( + f"base revision mismatch: expected {args.base_revision}, found {base_revision}") + try: + subprocess.run( + ["git", "-C", str(repo), "merge-base", "--is-ancestor", base_revision, revision], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["git", "-C", str(repo), "diff", "--quiet"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["git", "-C", str(repo), "diff", "--cached", "--quiet"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"candidate repository is not cleanly based on {base_revision}: {error}") from error + diff = git_output(repo, "diff", "--binary", "--no-ext-diff", base_revision, revision, "--") + diff_sha256 = hashlib.sha256(diff).hexdigest() + if diff_sha256 != args.candidate_diff_sha256: + raise PreflightError( + f"candidate diff SHA-256 mismatch: expected {args.candidate_diff_sha256}, found {diff_sha256}") + return { + "repository": REPOSITORY, + "revision": revision, + "base_revision": base_revision, + "diff_sha256": diff_sha256, + "executable_sha256": exporter_sha256, + } + + +def bind_candidate_attestation(output: Path, attestation: dict[str, str]) -> None: + manifest_path = output / "manifest.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot bind candidate attestation: {error}") from error + manifest["candidate"] = attestation + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +def build_command(args: argparse.Namespace, exporter: Path, output: Path) -> list[str]: + return [ + str(exporter), + "-m", str(resolved(args.model)), + "-bf", str(resolved(args.prompt)), + "-o", str(output), + "-c", str(args.context), + "-n", str(args.decode_steps), + "-b", str(args.batch), + "-ub", str(args.ubatch), + "-ngl", str(args.gpu_layers), + "-fa", "on", + "-ctk", "f16", + "-ctv", "f16", + "--expert-cache-slots", str(args.expert_cache_slots), + "--expert-cache-mib", str(args.expert_cache_mib), + ] def main() -> int: parser = argparse.ArgumentParser(description="Fail-closed launcher for llama.cpp DeepSeek V4.1 traces") parser.add_argument("--exporter", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--candidate-revision", required=True) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--candidate-diff-sha256", required=True) + parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) + parser.add_argument("--corpus-sha256", required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--prompt", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) @@ -31,6 +121,8 @@ def main() -> int: args = parser.parse_args() try: + if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: + raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") audit = run_preflight( model=args.model, prompt=args.prompt, @@ -56,6 +148,10 @@ def main() -> int: if not exporter.is_file() or not os.access(exporter, os.X_OK): raise PreflightError(f"trace exporter is not executable: {exporter}") exporter_sha256 = sha256_file(exporter) + model_sha256 = sha256_file(resolved(args.model)) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + attestation = candidate_attestation(args, exporter_sha256) output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") @@ -64,33 +160,30 @@ def main() -> int: environment["DSV41_TRACE_MEMORY_AUDIT"] = audits["memory"] environment["DSV41_TRACE_SWAP_AUDIT"] = audits["swap"] environment["DSV41_TRACE_WATCHDOG_AUDIT"] = audits["watchdog"] - command = [ - str(exporter), - "-m", str(resolved(args.model)), - "-f", str(resolved(args.prompt)), - "-o", str(output), - "-c", str(args.context), - "-n", str(args.decode_steps), - "-b", str(args.batch), - "-ub", str(args.ubatch), - "-ngl", str(args.gpu_layers), - "-fa", "on", - "-ctk", "f16", - "-ctv", "f16", - "--expert-cache-slots", str(args.expert_cache_slots), - "--expert-cache-mib", str(args.expert_cache_mib), - ] + command = build_command(args, exporter, output) print("exec:", shlex.join(command), file=sys.stderr) result = subprocess.run(command, env=environment, check=False) if result.returncode != 0: return result.returncode + run_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + watchdog_pid_file=args.watchdog_pid_file, + busy_patterns=args.busy_pattern, + ) + bind_embedded_audits(output, audits) + bind_prompt_provenance(output, args.corpus_name, args.corpus_sha256) + bind_candidate_attestation(output, attestation) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "llama.cpp": raise PreflightError("llama exporter wrote a non-llama.cpp trace") if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: raise PreflightError("llama trace build SHA-256 does not match the executed exporter") + if bundle.manifest.get("model", {}).get("sha256") != MODEL_SHA256: + raise PreflightError("llama trace model SHA-256 does not match the published GGUF") return 0 - except PreflightError as error: + except (PreflightError, TraceError) as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index 8588840232f7..53bd0325dd21 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 import argparse -import hashlib import json +import os import shutil import subprocess import sys from pathlib import Path -from preflight import PreflightError, require_nvme_path, resolved -from trace_format import TraceBundle, report +from preflight import PreflightError, require_nvme_path, resolved, run_preflight +from trace_format import CORPUS_SHA256, MODEL_SHA256, TraceBundle, report, sha256_file CORPORA = ( "correctness-prose.txt", @@ -26,12 +26,37 @@ def run(command: list[str]) -> None: raise RuntimeError(f"command failed with status {result.returncode}") -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() +def prepare_prompt( + *, + builder: Path, + model: Path, + corpus: Path, + output: Path, + target_tokens: int, +) -> dict[str, object]: + command = [ + str(builder), + "--model", str(model), + "--corpus", str(corpus), + "--output", str(output), + "--tokens", str(target_tokens), + ] + print("exec:", " ".join(command), file=sys.stderr) + result = subprocess.run(command, check=False, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"prompt builder failed: {result.stderr.strip()}") + try: + record = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise RuntimeError(f"prompt builder returned invalid JSON: {error}") from error + if record.get("actual_tokens") != target_tokens: + raise RuntimeError("prompt builder did not produce the requested token count") + record.update({ + "path": str(output), + "sha256": sha256_file(output), + "builder_sha256": sha256_file(builder), + }) + return record def main() -> int: @@ -42,6 +67,10 @@ def main() -> int: parser.add_argument("--watchdog-pid-file", type=Path, required=True) parser.add_argument("--llama-runner", type=Path, required=True) parser.add_argument("--llama-exporter", type=Path, required=True) + parser.add_argument("--llama-prompt-builder", type=Path, required=True) + parser.add_argument("--candidate-revision", required=True) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--candidate-diff-sha256", required=True) parser.add_argument("--ds4-runner", type=Path, required=True) parser.add_argument("--ds4-exporter", type=Path, required=True) parser.add_argument("--ds4-exporter-sha256", required=True) @@ -52,48 +81,94 @@ def main() -> int: parser.add_argument("--batch", type=int, default=2048) parser.add_argument("--expert-cache-slots", type=int, required=True) parser.add_argument("--expert-cache-mib", type=int, required=True) + parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) args = parser.parse_args() try: repo = resolved(args.repo) output = require_nvme_path(args.output, "matrix output") model = require_nvme_path(args.model, "model") + if not model.is_file(): + raise PreflightError(f"model is not a file: {model}") + model_sha256 = sha256_file(model) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + prompt_builder = resolved(args.llama_prompt_builder) + if not prompt_builder.is_file() or not os.access(prompt_builder, os.X_OK): + raise PreflightError(f"prompt builder is not executable: {prompt_builder}") if output.exists() and any(output.iterdir()): raise PreflightError(f"matrix output directory is not empty: {output}") inputs = output / "inputs" - inputs.mkdir(parents=True, exist_ok=True) + sources = inputs / "sources" + prompts = inputs / "prompts" + sources.mkdir(parents=True, exist_ok=True) + prompts.mkdir(parents=True, exist_ok=True) corpus_records = [] for name in CORPORA: - source = repo / "tests" / "corpus" / name + source = require_nvme_path(repo / "tests" / "corpus" / name, "repository corpus") if not source.is_file(): raise PreflightError(f"repository corpus is missing: {source}") - destination = inputs / name + destination = sources / name shutil.copyfile(source, destination) + source_sha256 = sha256_file(destination) + if source_sha256 != CORPUS_SHA256[name]: + raise PreflightError( + f"repository corpus SHA-256 mismatch for {name}: expected {CORPUS_SHA256[name]}, found {source_sha256}") corpus_records.append({ "name": name, "source": str(source), "path": str(destination), "byte_count": destination.stat().st_size, - "sha256": sha256_file(destination), + "sha256": source_sha256, }) results = [] + prompt_records = [] for context in args.contexts: if context < 32768 or context > 131072: raise PreflightError(f"context is outside the supported 32768..131072 matrix: {context}") + target_tokens = context - args.decode_steps + if target_tokens < 1: + raise PreflightError("decode steps leave no room for prompt tokens") + prepared_prompts = {} + for corpus in corpus_records: + stem = Path(corpus["name"]).stem + prompt = prompts / f"{stem}-c{context}.txt" + run_preflight( + model=model, + prompt=Path(corpus["path"]), + output=prompt, + watchdog_pid_file=args.watchdog_pid_file, + busy_patterns=args.busy_pattern, + ) + prepared = prepare_prompt( + builder=resolved(args.llama_prompt_builder), + model=model, + corpus=Path(corpus["path"]), + output=prompt, + target_tokens=target_tokens, + ) + prepared.update({"corpus": corpus["name"], "context": context}) + prepared_prompts[corpus["name"]] = prepared + prompt_records.append(prepared) for ubatch in args.ubatches: for corpus in corpus_records: stem = Path(corpus["name"]).stem case = f"{stem}-c{context}-ub{ubatch}" llama_output = output / "llama" / case ds4_output = output / "ds4" / case + prompt = prepared_prompts[corpus["name"]]["path"] common = [ "--model", str(model), - "--prompt", corpus["path"], + "--prompt", prompt, + "--corpus-name", corpus["name"], + "--corpus-sha256", corpus["sha256"], "--watchdog-pid-file", str(resolved(args.watchdog_pid_file)), "--context", str(context), "--decode-steps", str(args.decode_steps), ] + for pattern in args.busy_pattern: + common.extend(["--busy-pattern", pattern]) run([ sys.executable, str(resolved(args.ds4_runner)), @@ -108,6 +183,10 @@ def main() -> int: sys.executable, str(resolved(args.llama_runner)), "--exporter", str(resolved(args.llama_exporter)), + "--repo", str(repo), + "--candidate-revision", args.candidate_revision, + "--base-revision", args.base_revision, + "--candidate-diff-sha256", args.candidate_diff_sha256, "--output", str(llama_output), "--batch", str(args.batch), "--ubatch", str(ubatch), @@ -129,10 +208,18 @@ def main() -> int: summary = { "status": "TARGET PASS", "model": str(model), + "model_sha256": model_sha256, + "candidate_revision": args.candidate_revision, + "base_revision": args.base_revision, + "candidate_diff_sha256": args.candidate_diff_sha256, "corpora": corpus_records, + "prompts": prompt_records, "contexts": args.contexts, "ubatches": args.ubatches, "decode_steps": args.decode_steps, + "target_prompt_tokens": { + str(context): context - args.decode_steps for context in args.contexts + }, "cases": results, } (output / "summary.json").write_text( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 535f50a5fe2f..15e296b9ecb8 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -13,6 +13,16 @@ TRACE_FORMAT = "dsv41-trace" TRACE_VERSION = 1 +DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" +MODEL_SHA256 = "1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42" +REPOSITORY = "halo-box/strix-llama.cpp" +SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 +CORPUS_SHA256 = { + "correctness-prose.txt": "2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2", + "correctness-code.txt": "41b4246ef4e6b4e3f9f23a3d02aa8cdab48f495b3af0ebeaccea255679c771f0", + "correctness-structured.txt": "1278707adea5a953196c4cf5c04de301952813be3eac416a6aac4ff94f42f701", + "correctness-numeric.txt": "ebd444cf70662cc09289af45ef654af0953b98e967a449d031627d8ea92bc2e0", +} MANIFEST_NAME = "manifest.json" EVENTS_NAME = "events.jsonl" BLOBS_DIR = "blobs" @@ -28,6 +38,7 @@ } HARD_FAILURE_COMPONENTS = ( + "prompt.bytes", "prompt.tokens", "engram.row_ids", "expert.ids", @@ -65,6 +76,8 @@ class Mismatch: detail: str element_index: int | None = None byte_offset: int | None = None + token_index: int | None = None + component_element_index: int | None = None def as_dict(self) -> dict[str, Any]: result = { @@ -80,6 +93,10 @@ def as_dict(self) -> dict[str, Any]: result["element_index"] = self.element_index if self.byte_offset is not None: result["byte_offset"] = self.byte_offset + if self.token_index is not None: + result["token_index"] = self.token_index + if self.component_element_index is not None: + result["component_element_index"] = self.component_element_index return result @@ -198,10 +215,22 @@ def validate_event(event: dict[str, Any]) -> None: raise TraceError(f"unsupported event version: {event['trace_version']!r}") if event["byte_order"] != "little": raise TraceError("trace blobs must use little-endian byte order") + if event["phase"] not in ("input", "prefill", "decode"): + raise TraceError("event phase is invalid") + if not isinstance(event["step"], int) or event["step"] < 0: + raise TraceError("event step is invalid") + if not isinstance(event["token_start"], int) or event["token_start"] < 0: + raise TraceError("event token_start is invalid") + if not isinstance(event["token_count"], int) or event["token_count"] <= 0: + raise TraceError("event token_count is invalid") + if event["layer"] is not None and (not isinstance(event["layer"], int) or event["layer"] < 0): + raise TraceError("event layer is invalid") + if not isinstance(event["shape"], list) or not event["shape"] or any(dim <= 0 for dim in event["shape"]): + raise TraceError("event shape dimensions must be nonzero") if event["byte_count"] != expected_bytes(event): raise TraceError("event byte_count does not match dtype and shape") digest = event["sha256"] - if not isinstance(digest, str) or len(digest) != 64: + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError("event sha256 is invalid") if event["blob"] != f"{BLOBS_DIR}/{digest}.bin": raise TraceError("event blob path is not content addressed") @@ -340,8 +369,12 @@ def _validate_manifest(self) -> None: raise TraceError(f"manifest is missing {key}") if not isinstance(self.manifest["runtime"], str) or not self.manifest["runtime"]: raise TraceError("manifest runtime is invalid") + if self.manifest["runtime"] not in ("ds4", "llama.cpp"): + raise TraceError("manifest runtime must be ds4 or llama.cpp") if not isinstance(self.manifest["revision"], str) or not self.manifest["revision"]: raise TraceError("manifest revision is invalid") + if self.manifest["runtime"] == "ds4" and self.manifest["revision"] != DS4_REVISION: + raise TraceError(f"ds4 revision must be {DS4_REVISION}") if not isinstance(self.manifest["build"], dict): raise TraceError("manifest build is invalid") if re.fullmatch(r"[0-9a-f]{64}", self.manifest["build"].get("sha256", "")) is None: @@ -354,21 +387,108 @@ def _validate_manifest(self) -> None: raise TraceError(f"manifest {section} SHA-256 is invalid") if not isinstance(self.manifest[section].get("byte_count"), int): raise TraceError(f"manifest {section} byte_count is invalid") + if self.manifest["model"]["sha256"] != MODEL_SHA256: + raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") + if self.manifest["model"].get("architecture") != "deepseek41": + raise TraceError("model architecture must be deepseek41") + corpus_name = self.manifest["prompt"].get("corpus_name") + if corpus_name not in CORPUS_SHA256: + raise TraceError("prompt corpus is not in the fixed correctness corpus set") + if self.manifest["prompt"].get("corpus_sha256") != CORPUS_SHA256[corpus_name]: + raise TraceError(f"prompt corpus SHA-256 is invalid for {corpus_name}") + if self.manifest["runtime"] == "llama.cpp": + candidate = self.manifest.get("candidate") + if not isinstance(candidate, dict): + raise TraceError("llama.cpp candidate attestation is missing") + if candidate.get("repository") != REPOSITORY: + raise TraceError(f"candidate repository must be {REPOSITORY}") + for key in ("revision", "base_revision", "diff_sha256", "executable_sha256"): + if re.fullmatch(r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", + candidate.get(key, "")) is None: + raise TraceError(f"candidate {key} is invalid") + if not candidate["revision"].startswith(self.manifest["revision"]): + raise TraceError("candidate revision does not match the exporter build revision") + if candidate["executable_sha256"] != self.manifest["build"]["sha256"]: + raise TraceError("candidate executable SHA-256 does not match the trace build") for section in ("config", "comparison", "environment", "audits"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") + expected_config = { + "layer_count": 40, + "vocab_size": 129280, + "engram_layers": [1, 14], + "engram_rows_per_token": 4, + "expert_count": 384, + "experts_used": 6, + "candidate_source_layer": 20, + "candidate_topk_blocks": 2048, + "candidate_block_size": 8, + "index_top_k": 512, + "candidate_propagation_layers": [24, 28, 32, 36], + } + if self.manifest["config"].get("deepseek41") != expected_config: + raise TraceError("DeepSeek V4.1 configuration is invalid") if self.manifest["comparison"].get("logits") != "byte-identical-f32": raise TraceError("logit comparison policy must be byte-identical-f32") for kind in ("memory", "swap", "watchdog"): audit = self.manifest["audits"].get(kind) if not isinstance(audit, dict): raise TraceError(f"manifest {kind} audit reference is invalid") - if not isinstance(audit.get("path"), str) or not audit["path"]: + audit_path = audit.get("path") + if not isinstance(audit_path, str) or not audit_path: raise TraceError(f"manifest {kind} audit path is invalid") - if re.fullmatch(r"[0-9a-f]{64}", audit.get("sha256", "")) is None: + digest = audit.get("sha256", "") + if re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError(f"manifest {kind} audit SHA-256 is invalid") if not isinstance(audit.get("created_unix"), int) or audit["created_unix"] <= 0: raise TraceError(f"manifest {kind} audit timestamp is invalid") + expected_path = f"audits/{digest}.json" + if audit_path != expected_path: + raise TraceError(f"manifest {kind} audit path is not content addressed") + evidence_path = self.root / audit_path + try: + evidence = evidence_path.read_bytes() + except OSError as error: + raise TraceError(f"cannot read {kind} audit evidence: {error}") from error + if sha256_bytes(evidence) != digest: + raise TraceError(f"{kind} audit evidence SHA-256 mismatch") + try: + record = json.loads(evidence.decode("ascii")) + except (UnicodeError, json.JSONDecodeError) as error: + raise TraceError(f"{kind} audit evidence is invalid: {error}") from error + if record.get("kind") != kind or record.get("created_unix") != audit["created_unix"]: + raise TraceError(f"{kind} audit evidence metadata mismatch") + if not isinstance(record.get("data"), dict): + raise TraceError(f"{kind} audit evidence data is invalid") + if kind == "memory": + used = record["data"].get("mem_used_bytes") + if not isinstance(used, int) or used < 0 or used >= SOFT_MEMORY_LIMIT: + raise TraceError("memory audit evidence is invalid") + if kind == "swap": + if record["data"].get("enabled") is not False or record["data"].get("entries") != []: + raise TraceError("swap audit evidence does not report zero configured swap") + if kind == "watchdog": + required = ("pid", "start_time_ticks", "command_sha256", "heartbeat_path", "heartbeat_unix") + if any(key not in record["data"] for key in required): + raise TraceError("watchdog audit evidence is incomplete") + data = record["data"] + if not isinstance(data["pid"], int) or data["pid"] <= 1: + raise TraceError("watchdog audit PID is invalid") + if not isinstance(data["start_time_ticks"], int) or data["start_time_ticks"] <= 0: + raise TraceError("watchdog audit start time is invalid") + if not isinstance(data["command_sha256"], str) or re.fullmatch( + r"[0-9a-f]{64}", data["command_sha256"]) is None: + raise TraceError("watchdog audit command SHA-256 is invalid") + if not isinstance(data["heartbeat_path"], str) or not data["heartbeat_path"]: + raise TraceError("watchdog audit heartbeat path is invalid") + if not isinstance(data["heartbeat_unix"], int) or data["heartbeat_unix"] <= 0: + raise TraceError("watchdog audit heartbeat timestamp is invalid") + max_age = data.get("max_heartbeat_age_seconds") + if not isinstance(max_age, int) or max_age <= 0 or max_age > 30: + raise TraceError("watchdog audit heartbeat age is invalid") + if data["heartbeat_unix"] > record["created_unix"] or ( + record["created_unix"] - data["heartbeat_unix"] > max_age): + raise TraceError("watchdog audit heartbeat was stale when captured") def read_blob(self, event: dict[str, Any]) -> bytes: try: @@ -389,6 +509,8 @@ def _validate_coverage(self) -> None: raise TraceError("expected decode_steps is invalid") if self.manifest.get("config", {}).get("decode_steps") != decode_steps: raise TraceError("expected decode_steps does not match config") + for event in self.events: + self._validate_component_schema(event) if not isinstance(components, dict): raise TraceError("expected components are invalid") if self.manifest.get("model", {}).get("architecture") == "deepseek41": @@ -449,6 +571,66 @@ def _validate_coverage(self) -> None: if event["token_start"] != prompt_tokens + event["step"] or event["token_count"] != 1: raise TraceError(f"{component} decode token coordinates are invalid") + def _validate_component_schema(self, event: dict[str, Any]) -> None: + component = event["component"] + phase = event["phase"] + layer = event["layer"] + dtype = event["dtype"] + shape = event["shape"] + token_count = event["token_count"] + config = self.manifest["config"].get("deepseek41") + if not isinstance(config, dict): + raise TraceError("DeepSeek V4.1 component schema configuration is missing") + + if component == "prompt.bytes": + if phase != "input" or layer is not None or dtype != "bytes" or len(shape) != 1: + raise TraceError("prompt.bytes schema is invalid") + if shape[0] != self.manifest["prompt"]["byte_count"]: + raise TraceError("prompt.bytes length does not match the manifest") + if event["sha256"] != self.manifest["prompt"]["sha256"]: + raise TraceError("prompt.bytes SHA-256 does not match the manifest") + return + if component == "prompt.tokens": + if phase != "input" or layer is not None or dtype != "i32" or shape != [token_count]: + raise TraceError("prompt.tokens schema is invalid") + return + if component in ("logits.prefill", "logits.decode"): + if phase not in ("prefill", "decode") or layer is not None or dtype != "f32": + raise TraceError(f"{component} schema is invalid") + if token_count != 1 or shape != [config.get("vocab_size")]: + raise TraceError(f"{component} must contain one complete vocabulary-sized logit vector") + return + if component == "decode.greedy_token": + if phase != "decode" or layer is not None or dtype != "i32" or token_count != 1 or shape != [1]: + raise TraceError("decode.greedy_token schema is invalid") + return + + if phase not in ("prefill", "decode") or layer is None: + raise TraceError(f"{component} phase or layer is invalid") + if len(shape) != 2 or shape[1] != token_count: + raise TraceError(f"{component} second dimension must equal token_count") + widths = { + "engram.row_ids": ("i32", config.get("engram_rows_per_token")), + "expert.ids": ("i32", config.get("experts_used")), + "expert.weights": ("f32", config.get("experts_used")), + "attn.source": ("i32", None), + "attn.candidate_blocks": ("i32", None), + "attn.candidates": ("i32", config.get("index_top_k")), + } + if component not in widths: + raise TraceError(f"unsupported trace component: {component}") + expected_dtype, width = widths[component] + if dtype != expected_dtype: + raise TraceError(f"{component} dtype must be {expected_dtype}") + if width is not None and shape[0] != width: + raise TraceError(f"{component} shape must be [{width},token_count]") + if component == "attn.candidate_blocks" and shape[0] > config.get("candidate_topk_blocks", 0): + raise TraceError("attn.candidate_blocks width exceeds candidate_topk_blocks") + if component == "expert.ids": + values = struct.iter_unpack("= config.get("expert_count", 0) for value, in values): + raise TraceError("expert.ids contains an out-of-range original expert ID") + def first_byte_difference(left: bytes, right: bytes) -> int | None: for index, (a, b) in enumerate(zip(left, right)): @@ -465,8 +647,10 @@ def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: ("model.architecture", "model_identity"), ("prompt.sha256", "prompt_identity"), ("prompt.byte_count", "prompt_identity"), + ("expected.prompt_tokens", "tokenizer"), ("config.context", "configuration"), ("config.decode_steps", "configuration"), + ("config.deepseek41", "configuration"), ("comparison.logits", "comparison_policy"), ) for dotted, classification in checks: @@ -489,6 +673,26 @@ def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: + if left.root.resolve() == right.root.resolve(): + return Mismatch( + "artifact_identity", + "manifest", + "metadata", + -1, + -1, + None, + "cannot compare a trace bundle with itself", + ) + if left.manifest["runtime"] != "ds4" or right.manifest["runtime"] != "llama.cpp": + return Mismatch( + "runtime_role", + "manifest", + "metadata", + -1, + -1, + None, + "left trace must be pinned ds4 and right trace must be llama.cpp candidate", + ) mismatch = compare_manifests(left, right) if mismatch is not None: return mismatch @@ -553,6 +757,15 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: byte_offset = first_byte_difference(left_data, right_data) assert byte_offset is not None item_size = DTYPE_SIZES[left_event["dtype"]] + flat_element_index = byte_offset // item_size + token_index = None + component_element_index = None + token_count = left_event["token_count"] + elements = element_count(left_event["shape"]) + if token_count > 0 and elements % token_count == 0: + elements_per_token = elements // token_count + token_index = left_event["token_start"] + flat_element_index // elements_per_token + component_element_index = flat_element_index % elements_per_token detail = f"first byte mismatch at {byte_offset}" item_offset = byte_offset - byte_offset % item_size if item_offset + item_size <= min(len(left_data), len(right_data)): @@ -567,8 +780,10 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: template["token_start"], template["layer"], detail, - element_index=byte_offset // item_size, + element_index=flat_element_index, byte_offset=byte_offset, + token_index=token_index, + component_element_index=component_element_index, ) return None From 23a89dd3f2dc8051111e7a2ca1f50460d3238596 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 21:05:38 -0700 Subject: [PATCH 09/56] deepseek41 : complete trace attestations Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 138 +++++++++++---- tools/deepseek-v41-trace/README.md | 10 +- tools/deepseek-v41-trace/llama-trace.cpp | 4 +- tools/deepseek-v41-trace/preflight.py | 78 +++++++- tools/deepseek-v41-trace/run_ds4.py | 39 ++-- tools/deepseek-v41-trace/run_llama.py | 79 ++++++--- tools/deepseek-v41-trace/run_matrix.py | 25 ++- tools/deepseek-v41-trace/trace_format.py | 215 +++++++++++++++-------- 8 files changed, 423 insertions(+), 165 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 13c8a1fb2811..6d0a27cc0fb6 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -50,20 +50,42 @@ def audit_bytes(kind: str) -> bytes: return (json.dumps(AUDIT_RECORDS[kind], sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") -def manifest(runtime: str = "llama.cpp") -> dict: +def provenance_bytes(prompt: bytes = b"abc") -> bytes: + record = { + "format": "dsv41-prompt-provenance", + "version": 1, + "corpus_name": "correctness-prose.txt", + "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "model_sha256": trace.MODEL_SHA256, + "prompt_sha256": trace.sha256_bytes(prompt), + "prompt_byte_count": len(prompt), + "target_tokens": 2, + "actual_tokens": 2, + "builder_sha256": "8" * 64, + } + return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + +def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: + provenance_sha256 = trace.sha256_bytes(provenance_bytes(prompt)) result = { "runtime": runtime, "revision": trace.DS4_REVISION if runtime == "ds4" else "a" * 40, "build": {"sha256": "3" * 64}, "model": {"sha256": trace.MODEL_SHA256, "byte_count": 123, "architecture": "deepseek41"}, "prompt": { - "sha256": trace.sha256_bytes(b"abc"), - "byte_count": 3, + "sha256": trace.sha256_bytes(prompt), + "byte_count": len(prompt), "corpus_name": "correctness-prose.txt", "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "target_tokens": 2, + "provenance": { + "path": f"provenance/{provenance_sha256}.json", + "sha256": provenance_sha256, + }, }, "config": { - "context": 32768, + "context": 3, "decode_steps": 1, "batch": 512, "ubatch": 128, @@ -76,7 +98,7 @@ def manifest(runtime: str = "llama.cpp") -> dict: "layer_count": 40, "vocab_size": 129280, "engram_layers": [1, 14], - "engram_rows_per_token": 4, + "engram_rows_per_token": 24, "expert_count": 384, "experts_used": 6, "candidate_source_layer": 20, @@ -106,12 +128,15 @@ def manifest(runtime: str = "llama.cpp") -> dict: }, "environment": {}, "audits": { - kind: { - "path": f"audits/{trace.sha256_bytes(audit_bytes(kind))}.json", - "sha256": trace.sha256_bytes(audit_bytes(kind)), - "created_unix": 1, + phase: { + kind: { + "path": f"audits/{phase}/{trace.sha256_bytes(audit_bytes(kind))}.json", + "sha256": trace.sha256_bytes(audit_bytes(kind)), + "created_unix": 1, + } + for kind in ("memory", "swap", "watchdog") } - for kind in ("memory", "swap", "watchdog") + for phase in ("pre", "post") }, } if runtime == "llama.cpp": @@ -125,12 +150,17 @@ def manifest(runtime: str = "llama.cpp") -> dict: return result -def add_required_events(writer: object, logits: bytes | None = None) -> None: - audit_root = writer.root / "audits" - audit_root.mkdir(exist_ok=True) - for kind in ("memory", "swap", "watchdog"): - data = audit_bytes(kind) - (audit_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) +def add_required_events(writer: object, logits: bytes | None = None, prompt: bytes = b"abc") -> None: + for phase in ("pre", "post"): + audit_root = writer.root / "audits" / phase + audit_root.mkdir(parents=True, exist_ok=True) + for kind in ("memory", "swap", "watchdog"): + data = audit_bytes(kind) + (audit_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) + provenance_root = writer.root / "provenance" + provenance_root.mkdir(exist_ok=True) + data = provenance_bytes(prompt) + (provenance_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) writer.add_event( component="prompt.bytes", phase="input", @@ -139,8 +169,8 @@ def add_required_events(writer: object, logits: bytes | None = None) -> None: token_count=2, layer=None, dtype="bytes", - shape=[3], - data=b"abc", + shape=[len(prompt)], + data=prompt, ) writer.add_event( component="prompt.tokens", @@ -161,8 +191,8 @@ def add_required_events(writer: object, logits: bytes | None = None) -> None: token_count=2, layer=1, dtype="i32", - shape=[4, 2], - data=struct.pack(" None: ) writer.add_event( component="engram.row_ids", phase="decode", step=0, token_start=2, token_count=1, - layer=1, dtype="i32", shape=[4, 1], data=struct.pack(" None: ) writer.add_event( component="engram.row_ids", phase="prefill", step=0, token_start=0, token_count=2, - layer=14, dtype="i32", shape=[4, 2], data=struct.pack(" None: self.assertEqual(divergence["token_index"], 1) self.assertEqual(divergence["component_element_index"], 2) + def test_compare_selects_global_first_token_divergence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: + add_required_events(writer) + events_path = right / trace.EVENTS_NAME + events = [json.loads(line) for line in events_path.read_text(encoding="ascii").splitlines()] + for layer, element in ((0, 8), (1, 2)): + event = next(item for item in events if ( + item["component"] == "expert.ids" and + item["phase"] == "prefill" and + item["layer"] == layer + )) + values = list(struct.unpack(" None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -418,7 +479,7 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" bad_manifest = manifest() - bad_manifest["audits"]["watchdog"] = "watchdog.json" + bad_manifest["audits"]["pre"]["watchdog"] = "watchdog.json" with trace.TraceBundleWriter(root, bad_manifest) as writer: add_required_events(writer) with self.assertRaisesRegex(trace.TraceError, "watchdog audit reference"): @@ -429,8 +490,17 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: trace_manifest = manifest() with trace.TraceBundleWriter(root, trace_manifest) as writer: add_required_events(writer) - (root / trace_manifest["audits"]["memory"]["path"]).unlink() - with self.assertRaisesRegex(trace.TraceError, "memory audit evidence"): + (root / trace_manifest["audits"]["pre"]["memory"]["path"]).unlink() + with self.assertRaisesRegex(trace.TraceError, "pre memory audit evidence"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest() + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + (root / trace_manifest["audits"]["post"]["watchdog"]["path"]).unlink() + with self.assertRaisesRegex(trace.TraceError, "post watchdog audit evidence"): trace.TraceBundle(root) def test_rejects_unpinned_ds4_revision(self) -> None: @@ -510,23 +580,12 @@ def test_manifest_mismatch_is_classified(self) -> None: left = Path(temp) / "left" right = Path(temp) / "right" left_manifest = manifest("ds4") - right_manifest = manifest("llama.cpp") right_prompt = b"abd" - right_manifest["prompt"]["sha256"] = trace.sha256_bytes(right_prompt) + right_manifest = manifest("llama.cpp", right_prompt) with trace.TraceBundleWriter(left, left_manifest) as writer: add_required_events(writer) with trace.TraceBundleWriter(right, right_manifest) as writer: - add_required_events(writer) - events_path = right / trace.EVENTS_NAME - events = [json.loads(line) for line in events_path.read_text(encoding="ascii").splitlines()] - prompt_event = next(event for event in events if event["component"] == "prompt.bytes") - prompt_event["sha256"] = trace.sha256_bytes(right_prompt) - prompt_event["blob"] = f"blobs/{prompt_event['sha256']}.bin" - (right / prompt_event["blob"]).write_bytes(right_prompt) - events_path.write_text( - "".join(trace.canonical_json(event) + "\n" for event in events), - encoding="ascii", - ) + add_required_events(writer, prompt=right_prompt) result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) self.assertEqual(result["first_divergence"]["classification"], "prompt_identity") @@ -534,6 +593,7 @@ def test_identically_incomplete_decode_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" incomplete = manifest() + incomplete["config"]["context"] = 4 incomplete["config"]["decode_steps"] = 2 incomplete["expected"]["decode_steps"] = 2 with trace.TraceBundleWriter(root, incomplete) as writer: diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index ee76ae51590d..5a93c509d870 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -7,11 +7,12 @@ Each trace is a directory: - `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, and content-addressed memory/swap/watchdog audit references. - `events.jsonl` is an ordered stream of content-addressed event records. - `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. -- `audits/.json` stores the immutable safety evidence referenced by the manifest. +- `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. +- `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. -Internal tensors use raw ggml dimension order. The validator requires Engram rows as i32 `[4, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with `token_count` in the second dimension, layer-20 candidate blocks as rank-2 i32 with width at most 2048, propagated candidates as i32 `[512, token_count]`, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. +Internal tensors use raw ggml dimension order. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with width at most 512 and `token_count` in the second dimension, layer-20 candidate blocks as rank-2 i32 with width at most 2048, propagated candidates as rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. Validate or compare bundles: @@ -55,6 +56,7 @@ python3 tools/deepseek-v41-trace/run_ds4.py \ --prompt /path/on/nvme/correctness-prose-32768.txt \ --corpus-name correctness-prose.txt \ --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ + --prompt-provenance /path/on/nvme/correctness-prose-32768.txt.provenance.json \ --output /path/on/nvme/traces/ds4-prose-32768 \ --watchdog-pid-file /run/user/$(id -u)/dsv41-watchdog.pid \ --exporter /path/to/pinned-ds4-trace-exporter \ @@ -65,6 +67,6 @@ The exporter is intentionally external to the canonical ds4 checkout. It must be The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and embeds content-addressed audit evidence in the trace. It requires the exact candidate revision, full-graph base revision, expected base-to-candidate binary diff SHA-256, and repository path. It rejects a dirty checkout or an exporter whose embedded build revision or executable hash does not match that attestation. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact candidate revision, full-graph base revision, expected base-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision or executable hash does not match that attestation. -`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, records their hashes, builds exact-length prompt artifacts, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build. Its default context matrix is 32768. Pass `--contexts 32768 65536 98304 131072` only after the 32K target passes. +`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the candidate revision, full-graph base revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass `--contexts 32768 65536 98304 131072` only after the 32K target passes. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 925e4c9520f1..ed70fbb033f7 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -209,7 +209,7 @@ static std::string tensor_dtype(const ggml_tensor * tensor) { static std::vector tensor_shape(const ggml_tensor * tensor) { int rank = GGML_MAX_DIMS; - while (rank > 1 && tensor->ne[rank - 1] == 1) { + while (rank > 2 && tensor->ne[rank - 1] == 1) { --rank; } std::vector result; @@ -562,7 +562,7 @@ int main(int argc, char ** argv) { {"layer_count", 40}, {"vocab_size", n_vocab}, {"engram_layers", {1, 14}}, - {"engram_rows_per_token", 4}, + {"engram_rows_per_token", 24}, {"expert_count", 384}, {"experts_used", 6}, {"candidate_source_layer", 20}, diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index 7649352cbfde..c66be6627ee8 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -215,9 +215,9 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: return result -def embed_audits(trace_root: Path, audits: dict[str, str]) -> dict[str, dict[str, object]]: +def embed_audits(trace_root: Path, phase: str, audits: dict[str, str]) -> dict[str, dict[str, object]]: trace_root = resolved(trace_root) - embedded_root = trace_root / "audits" + embedded_root = trace_root / "audits" / phase embedded_root.mkdir(parents=True, exist_ok=True) result = {} for kind in ("memory", "swap", "watchdog"): @@ -235,27 +235,72 @@ def embed_audits(trace_root: Path, audits: dict[str, str]) -> dict[str, dict[str except (UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: raise PreflightError(f"cannot embed {kind} audit: {error}") from error result[kind] = { - "path": f"audits/{digest}.json", + "path": f"audits/{phase}/{digest}.json", "sha256": digest, "created_unix": created, } return result -def bind_embedded_audits(trace_root: Path, audits: dict[str, str]) -> None: +def bind_embedded_audits(trace_root: Path, audit_sets: dict[str, dict[str, str]]) -> None: trace_root = resolved(trace_root) manifest_path = trace_root / "manifest.json" try: manifest = json.loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise PreflightError(f"cannot bind trace audits: {error}") from error - manifest["audits"] = embed_audits(trace_root, audits) + if set(audit_sets) != {"pre", "post"}: + raise PreflightError("trace requires pre and post audit sets") + manifest["audits"] = { + phase: embed_audits(trace_root, phase, audits) + for phase, audits in audit_sets.items() + } temp = manifest_path.with_suffix(".tmp") temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") os.replace(temp, manifest_path) -def bind_prompt_provenance(trace_root: Path, corpus_name: str, corpus_sha256: str) -> None: +def validate_prompt_provenance( + path: Path, + *, + prompt: Path, + corpus_name: str, + corpus_sha256: str, + model_sha256: str, + target_tokens: int, +) -> dict[str, object]: + path = require_nvme_path(path, "prompt provenance") + try: + data = path.read_bytes() + record = json.loads(data.decode("ascii")) + prompt_path = resolved(prompt) + prompt_bytes = prompt_path.read_bytes() + prompt_size = prompt_path.stat().st_size + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"prompt provenance is invalid: {error}") from error + if not isinstance(record, dict): + raise PreflightError("prompt provenance must be a JSON object") + expected = { + "format": "dsv41-prompt-provenance", + "version": 1, + "corpus_name": corpus_name, + "corpus_sha256": corpus_sha256, + "model_sha256": model_sha256, + "prompt_sha256": sha256_bytes(prompt_bytes), + "prompt_byte_count": prompt_size, + "target_tokens": target_tokens, + "actual_tokens": target_tokens, + } + for key, value in expected.items(): + if record.get(key) != value: + raise PreflightError(f"prompt provenance {key} mismatch") + builder_sha256 = record.get("builder_sha256", "") + if not isinstance(builder_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", builder_sha256) is None: + raise PreflightError("prompt provenance builder SHA-256 is invalid") + return {"path": str(path), "bytes": data, "record": record} + + +def bind_prompt_provenance(trace_root: Path, provenance: dict[str, object]) -> None: trace_root = resolved(trace_root) manifest_path = trace_root / "manifest.json" try: @@ -265,8 +310,25 @@ def bind_prompt_provenance(trace_root: Path, corpus_name: str, corpus_sha256: st prompt = manifest.get("prompt") if not isinstance(prompt, dict): raise PreflightError("trace manifest prompt is invalid") - prompt["corpus_name"] = corpus_name - prompt["corpus_sha256"] = corpus_sha256 + record = provenance["record"] + data = provenance["bytes"] + if not isinstance(record, dict) or not isinstance(data, bytes): + raise PreflightError("validated prompt provenance is invalid") + digest = sha256_bytes(data) + provenance_root = trace_root / "provenance" + provenance_root.mkdir(parents=True, exist_ok=True) + destination = provenance_root / f"{digest}.json" + if destination.exists() and destination.read_bytes() != data: + raise PreflightError(f"content-addressed provenance collision: {destination}") + if not destination.exists(): + destination.write_bytes(data) + prompt["corpus_name"] = record["corpus_name"] + prompt["corpus_sha256"] = record["corpus_sha256"] + prompt["target_tokens"] = record["target_tokens"] + prompt["provenance"] = { + "path": f"provenance/{digest}.json", + "sha256": digest, + } temp = manifest_path.with_suffix(".tmp") temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") os.replace(temp, manifest_path) diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index fcb27030f2e3..4dc743cca3eb 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -8,7 +8,15 @@ import sys from pathlib import Path -from preflight import PreflightError, bind_embedded_audits, bind_prompt_provenance, resolved, run_preflight, write_audits +from preflight import ( + PreflightError, + bind_embedded_audits, + bind_prompt_provenance, + resolved, + run_preflight, + validate_prompt_provenance, + write_audits, +) from trace_format import CORPUS_SHA256, MODEL_SHA256, TraceBundle, TraceError, sha256_file DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" @@ -62,6 +70,7 @@ def main() -> int: parser.add_argument("--exporter-sha256", required=True) parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) parser.add_argument("--corpus-sha256", required=True) + parser.add_argument("--prompt-provenance", type=Path, required=True) parser.add_argument("--context", type=int, default=32768) parser.add_argument("--decode-steps", type=int, default=8) parser.add_argument("--prefill-chunk", type=int, default=512) @@ -71,8 +80,8 @@ def main() -> int: try: if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") - audit = preflight(args) if args.preflight_only: + audit = preflight(args) print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) return 0 @@ -86,11 +95,20 @@ def main() -> int: model_sha256 = sha256_file(resolved(args.model)) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") - audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} + provenance = validate_prompt_provenance( + args.prompt_provenance, + prompt=args.prompt, + corpus_name=args.corpus_name, + corpus_sha256=args.corpus_sha256, + model_sha256=model_sha256, + target_tokens=args.context - args.decode_steps, + ) output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") - audits = write_audits(Path(str(output) + ".audit"), audit) + preflight_audit = preflight(args) + preflight_audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} + pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) command = [ str(exporter), "--model", str(resolved(args.model)), @@ -99,17 +117,18 @@ def main() -> int: "--context", str(args.context), "--decode-steps", str(args.decode_steps), "--prefill-chunk", str(args.prefill_chunk), - "--memory-audit", audits["memory"], - "--swap-audit", audits["swap"], - "--watchdog-audit", audits["watchdog"], + "--memory-audit", pre_audits["memory"], + "--swap-audit", pre_audits["swap"], + "--watchdog-audit", pre_audits["watchdog"], ] print("exec:", shlex.join(command), file=sys.stderr) result = subprocess.run(command, cwd=resolved(args.checkout), check=False) if result.returncode != 0: return result.returncode - preflight(args) - bind_embedded_audits(output, audits) - bind_prompt_provenance(output, args.corpus_name, args.corpus_sha256) + postflight_audit = preflight(args) + post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) + bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) + bind_prompt_provenance(output, provenance) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "ds4": raise PreflightError("ds4 exporter wrote a non-ds4 trace") diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index d1af2663b5e0..522ab868b8aa 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -9,7 +9,15 @@ import sys from pathlib import Path -from preflight import PreflightError, bind_embedded_audits, bind_prompt_provenance, resolved, run_preflight, write_audits +from preflight import ( + PreflightError, + bind_embedded_audits, + bind_prompt_provenance, + resolved, + run_preflight, + validate_prompt_provenance, + write_audits, +) from trace_format import CORPUS_SHA256, MODEL_SHA256, REPOSITORY, TraceBundle, TraceError, sha256_file @@ -51,6 +59,9 @@ def candidate_attestation(args: argparse.Namespace, exporter_sha256: str) -> dic ) except (OSError, subprocess.CalledProcessError) as error: raise PreflightError(f"candidate repository is not cleanly based on {base_revision}: {error}") from error + status = git_output(repo, "status", "--porcelain", "--untracked-files=all") + if status: + raise PreflightError("candidate repository has tracked or untracked changes") diff = git_output(repo, "diff", "--binary", "--no-ext-diff", base_revision, revision, "--") diff_sha256 = hashlib.sha256(diff).hexdigest() if diff_sha256 != args.candidate_diff_sha256: @@ -105,6 +116,7 @@ def main() -> int: parser.add_argument("--candidate-diff-sha256", required=True) parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) parser.add_argument("--corpus-sha256", required=True) + parser.add_argument("--prompt-provenance", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--prompt", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) @@ -123,24 +135,14 @@ def main() -> int: try: if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") - audit = run_preflight( - model=args.model, - prompt=args.prompt, - output=args.output, - watchdog_pid_file=args.watchdog_pid_file, - busy_patterns=args.busy_pattern, - ) - audit["runtime"] = "llama.cpp" - audit["config"] = { - "context": args.context, - "decode_steps": args.decode_steps, - "batch": args.batch, - "ubatch": args.ubatch, - "expert_cache_slots": args.expert_cache_slots, - "expert_cache_mib": args.expert_cache_mib, - "gpu_layers": args.gpu_layers, - } if args.preflight_only: + audit = run_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + watchdog_pid_file=args.watchdog_pid_file, + busy_patterns=args.busy_pattern, + ) print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) return 0 @@ -151,29 +153,56 @@ def main() -> int: model_sha256 = sha256_file(resolved(args.model)) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + provenance = validate_prompt_provenance( + args.prompt_provenance, + prompt=args.prompt, + corpus_name=args.corpus_name, + corpus_sha256=args.corpus_sha256, + model_sha256=model_sha256, + target_tokens=args.context - args.decode_steps, + ) attestation = candidate_attestation(args, exporter_sha256) output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") - audits = write_audits(Path(str(output) + ".audit"), audit) + preflight_audit = run_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + watchdog_pid_file=args.watchdog_pid_file, + busy_patterns=args.busy_pattern, + ) + preflight_audit["runtime"] = "llama.cpp" + preflight_audit["config"] = { + "context": args.context, + "decode_steps": args.decode_steps, + "batch": args.batch, + "ubatch": args.ubatch, + "expert_cache_slots": args.expert_cache_slots, + "expert_cache_mib": args.expert_cache_mib, + "gpu_layers": args.gpu_layers, + } + pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) environment = os.environ.copy() - environment["DSV41_TRACE_MEMORY_AUDIT"] = audits["memory"] - environment["DSV41_TRACE_SWAP_AUDIT"] = audits["swap"] - environment["DSV41_TRACE_WATCHDOG_AUDIT"] = audits["watchdog"] + environment["DSV41_TRACE_MEMORY_AUDIT"] = pre_audits["memory"] + environment["DSV41_TRACE_SWAP_AUDIT"] = pre_audits["swap"] + environment["DSV41_TRACE_WATCHDOG_AUDIT"] = pre_audits["watchdog"] command = build_command(args, exporter, output) print("exec:", shlex.join(command), file=sys.stderr) result = subprocess.run(command, env=environment, check=False) if result.returncode != 0: return result.returncode - run_preflight( + postflight_audit = run_preflight( model=args.model, prompt=args.prompt, output=args.output, watchdog_pid_file=args.watchdog_pid_file, busy_patterns=args.busy_pattern, ) - bind_embedded_audits(output, audits) - bind_prompt_provenance(output, args.corpus_name, args.corpus_sha256) + postflight_audit["runtime"] = "llama.cpp" + post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) + bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) + bind_prompt_provenance(output, provenance) bind_candidate_attestation(output, attestation) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "llama.cpp": diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index 53bd0325dd21..f8209d1337a9 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -31,6 +31,8 @@ def prepare_prompt( builder: Path, model: Path, corpus: Path, + corpus_name: str, + corpus_sha256: str, output: Path, target_tokens: int, ) -> dict[str, object]: @@ -52,10 +54,25 @@ def prepare_prompt( if record.get("actual_tokens") != target_tokens: raise RuntimeError("prompt builder did not produce the requested token count") record.update({ - "path": str(output), - "sha256": sha256_file(output), + "format": "dsv41-prompt-provenance", + "version": 1, + "corpus_name": corpus_name, + "corpus_sha256": corpus_sha256, + "model_sha256": MODEL_SHA256, + "prompt_sha256": sha256_file(output), + "prompt_byte_count": output.stat().st_size, "builder_sha256": sha256_file(builder), }) + provenance_path = output.with_suffix(output.suffix + ".provenance.json") + provenance_path.write_text( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + record.update({ + "path": str(output), + "provenance_path": str(provenance_path), + "provenance_sha256": sha256_file(provenance_path), + }) return record @@ -145,6 +162,8 @@ def main() -> int: builder=resolved(args.llama_prompt_builder), model=model, corpus=Path(corpus["path"]), + corpus_name=corpus["name"], + corpus_sha256=corpus["sha256"], output=prompt, target_tokens=target_tokens, ) @@ -158,9 +177,11 @@ def main() -> int: llama_output = output / "llama" / case ds4_output = output / "ds4" / case prompt = prepared_prompts[corpus["name"]]["path"] + provenance = prepared_prompts[corpus["name"]]["provenance_path"] common = [ "--model", str(model), "--prompt", prompt, + "--prompt-provenance", provenance, "--corpus-name", corpus["name"], "--corpus-sha256", corpus["sha256"], "--watchdog-pid-file", str(resolved(args.watchdog_pid_file)), diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 15e296b9ecb8..2e10a69a0e05 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -377,7 +377,8 @@ def _validate_manifest(self) -> None: raise TraceError(f"ds4 revision must be {DS4_REVISION}") if not isinstance(self.manifest["build"], dict): raise TraceError("manifest build is invalid") - if re.fullmatch(r"[0-9a-f]{64}", self.manifest["build"].get("sha256", "")) is None: + build_sha256 = self.manifest["build"].get("sha256", "") + if not isinstance(build_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", build_sha256) is None: raise TraceError("manifest build SHA-256 is invalid") for section in ("model", "prompt"): if not isinstance(self.manifest[section], dict): @@ -385,8 +386,11 @@ def _validate_manifest(self) -> None: digest = self.manifest[section].get("sha256") if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError(f"manifest {section} SHA-256 is invalid") - if not isinstance(self.manifest[section].get("byte_count"), int): + if not isinstance(self.manifest[section].get("byte_count"), int) or self.manifest[section]["byte_count"] <= 0: raise TraceError(f"manifest {section} byte_count is invalid") + for section in ("config", "comparison", "environment", "audits"): + if not isinstance(self.manifest[section], dict): + raise TraceError(f"manifest {section} is invalid") if self.manifest["model"]["sha256"] != MODEL_SHA256: raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") if self.manifest["model"].get("architecture") != "deepseek41": @@ -396,6 +400,41 @@ def _validate_manifest(self) -> None: raise TraceError("prompt corpus is not in the fixed correctness corpus set") if self.manifest["prompt"].get("corpus_sha256") != CORPUS_SHA256[corpus_name]: raise TraceError(f"prompt corpus SHA-256 is invalid for {corpus_name}") + provenance = self.manifest["prompt"].get("provenance") + if not isinstance(provenance, dict): + raise TraceError("prompt provenance reference is missing") + provenance_sha256 = provenance.get("sha256", "") + if not isinstance(provenance_sha256, str) or re.fullmatch( + r"[0-9a-f]{64}", provenance_sha256) is None: + raise TraceError("prompt provenance SHA-256 is invalid") + if provenance.get("path") != f"provenance/{provenance_sha256}.json": + raise TraceError("prompt provenance path is not content addressed") + try: + provenance_bytes = (self.root / provenance["path"]).read_bytes() + provenance_record = json.loads(provenance_bytes.decode("ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise TraceError(f"cannot read prompt provenance: {error}") from error + if sha256_bytes(provenance_bytes) != provenance_sha256: + raise TraceError("prompt provenance SHA-256 mismatch") + expected_target = self.manifest["config"].get("context", 0) - self.manifest["config"].get("decode_steps", 0) + provenance_checks = { + "format": "dsv41-prompt-provenance", + "version": 1, + "corpus_name": corpus_name, + "corpus_sha256": self.manifest["prompt"]["corpus_sha256"], + "model_sha256": self.manifest["model"]["sha256"], + "prompt_sha256": self.manifest["prompt"]["sha256"], + "prompt_byte_count": self.manifest["prompt"]["byte_count"], + "target_tokens": expected_target, + "actual_tokens": expected_target, + } + for key, value in provenance_checks.items(): + if provenance_record.get(key) != value: + raise TraceError(f"prompt provenance {key} mismatch") + if re.fullmatch(r"[0-9a-f]{64}", provenance_record.get("builder_sha256", "")) is None: + raise TraceError("prompt provenance builder SHA-256 is invalid") + if self.manifest["prompt"].get("target_tokens") != expected_target: + raise TraceError("prompt target token count does not fill the configured context") if self.manifest["runtime"] == "llama.cpp": candidate = self.manifest.get("candidate") if not isinstance(candidate, dict): @@ -403,21 +442,19 @@ def _validate_manifest(self) -> None: if candidate.get("repository") != REPOSITORY: raise TraceError(f"candidate repository must be {REPOSITORY}") for key in ("revision", "base_revision", "diff_sha256", "executable_sha256"): - if re.fullmatch(r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", - candidate.get(key, "")) is None: + value = candidate.get(key, "") + if not isinstance(value, str) or re.fullmatch( + r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: raise TraceError(f"candidate {key} is invalid") if not candidate["revision"].startswith(self.manifest["revision"]): raise TraceError("candidate revision does not match the exporter build revision") if candidate["executable_sha256"] != self.manifest["build"]["sha256"]: raise TraceError("candidate executable SHA-256 does not match the trace build") - for section in ("config", "comparison", "environment", "audits"): - if not isinstance(self.manifest[section], dict): - raise TraceError(f"manifest {section} is invalid") expected_config = { "layer_count": 40, "vocab_size": 129280, "engram_layers": [1, 14], - "engram_rows_per_token": 4, + "engram_rows_per_token": 24, "expert_count": 384, "experts_used": 6, "candidate_source_layer": 20, @@ -430,65 +467,71 @@ def _validate_manifest(self) -> None: raise TraceError("DeepSeek V4.1 configuration is invalid") if self.manifest["comparison"].get("logits") != "byte-identical-f32": raise TraceError("logit comparison policy must be byte-identical-f32") - for kind in ("memory", "swap", "watchdog"): - audit = self.manifest["audits"].get(kind) - if not isinstance(audit, dict): - raise TraceError(f"manifest {kind} audit reference is invalid") - audit_path = audit.get("path") - if not isinstance(audit_path, str) or not audit_path: - raise TraceError(f"manifest {kind} audit path is invalid") - digest = audit.get("sha256", "") - if re.fullmatch(r"[0-9a-f]{64}", digest) is None: - raise TraceError(f"manifest {kind} audit SHA-256 is invalid") - if not isinstance(audit.get("created_unix"), int) or audit["created_unix"] <= 0: - raise TraceError(f"manifest {kind} audit timestamp is invalid") - expected_path = f"audits/{digest}.json" - if audit_path != expected_path: - raise TraceError(f"manifest {kind} audit path is not content addressed") - evidence_path = self.root / audit_path - try: - evidence = evidence_path.read_bytes() - except OSError as error: - raise TraceError(f"cannot read {kind} audit evidence: {error}") from error - if sha256_bytes(evidence) != digest: - raise TraceError(f"{kind} audit evidence SHA-256 mismatch") - try: - record = json.loads(evidence.decode("ascii")) - except (UnicodeError, json.JSONDecodeError) as error: - raise TraceError(f"{kind} audit evidence is invalid: {error}") from error - if record.get("kind") != kind or record.get("created_unix") != audit["created_unix"]: - raise TraceError(f"{kind} audit evidence metadata mismatch") - if not isinstance(record.get("data"), dict): - raise TraceError(f"{kind} audit evidence data is invalid") - if kind == "memory": - used = record["data"].get("mem_used_bytes") - if not isinstance(used, int) or used < 0 or used >= SOFT_MEMORY_LIMIT: - raise TraceError("memory audit evidence is invalid") - if kind == "swap": - if record["data"].get("enabled") is not False or record["data"].get("entries") != []: - raise TraceError("swap audit evidence does not report zero configured swap") - if kind == "watchdog": - required = ("pid", "start_time_ticks", "command_sha256", "heartbeat_path", "heartbeat_unix") - if any(key not in record["data"] for key in required): - raise TraceError("watchdog audit evidence is incomplete") - data = record["data"] - if not isinstance(data["pid"], int) or data["pid"] <= 1: - raise TraceError("watchdog audit PID is invalid") - if not isinstance(data["start_time_ticks"], int) or data["start_time_ticks"] <= 0: - raise TraceError("watchdog audit start time is invalid") - if not isinstance(data["command_sha256"], str) or re.fullmatch( - r"[0-9a-f]{64}", data["command_sha256"]) is None: - raise TraceError("watchdog audit command SHA-256 is invalid") - if not isinstance(data["heartbeat_path"], str) or not data["heartbeat_path"]: - raise TraceError("watchdog audit heartbeat path is invalid") - if not isinstance(data["heartbeat_unix"], int) or data["heartbeat_unix"] <= 0: - raise TraceError("watchdog audit heartbeat timestamp is invalid") - max_age = data.get("max_heartbeat_age_seconds") - if not isinstance(max_age, int) or max_age <= 0 or max_age > 30: - raise TraceError("watchdog audit heartbeat age is invalid") - if data["heartbeat_unix"] > record["created_unix"] or ( - record["created_unix"] - data["heartbeat_unix"] > max_age): - raise TraceError("watchdog audit heartbeat was stale when captured") + for audit_phase in ("pre", "post"): + phase_audits = self.manifest["audits"].get(audit_phase) + if not isinstance(phase_audits, dict): + raise TraceError(f"manifest {audit_phase} audit set is invalid") + for kind in ("memory", "swap", "watchdog"): + self._validate_audit_reference(audit_phase, kind, phase_audits.get(kind)) + + def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: + if not isinstance(audit, dict): + raise TraceError(f"manifest {phase} {kind} audit reference is invalid") + audit_path = audit.get("path") + if not isinstance(audit_path, str) or not audit_path: + raise TraceError(f"manifest {phase} {kind} audit path is invalid") + digest = audit.get("sha256", "") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError(f"manifest {phase} {kind} audit SHA-256 is invalid") + if not isinstance(audit.get("created_unix"), int) or audit["created_unix"] <= 0: + raise TraceError(f"manifest {phase} {kind} audit timestamp is invalid") + expected_path = f"audits/{phase}/{digest}.json" + if audit_path != expected_path: + raise TraceError(f"manifest {phase} {kind} audit path is not content addressed") + evidence_path = self.root / audit_path + try: + evidence = evidence_path.read_bytes() + except OSError as error: + raise TraceError(f"cannot read {phase} {kind} audit evidence: {error}") from error + if sha256_bytes(evidence) != digest: + raise TraceError(f"{phase} {kind} audit evidence SHA-256 mismatch") + try: + record = json.loads(evidence.decode("ascii")) + except (UnicodeError, json.JSONDecodeError) as error: + raise TraceError(f"{phase} {kind} audit evidence is invalid: {error}") from error + if record.get("kind") != kind or record.get("created_unix") != audit["created_unix"]: + raise TraceError(f"{phase} {kind} audit evidence metadata mismatch") + if not isinstance(record.get("data"), dict): + raise TraceError(f"{phase} {kind} audit evidence data is invalid") + if kind == "memory": + used = record["data"].get("mem_used_bytes") + if not isinstance(used, int) or used < 0 or used >= SOFT_MEMORY_LIMIT: + raise TraceError(f"{phase} memory audit evidence is invalid") + if kind == "swap": + if record["data"].get("enabled") is not False or record["data"].get("entries") != []: + raise TraceError(f"{phase} swap audit evidence does not report zero configured swap") + if kind == "watchdog": + required = ("pid", "start_time_ticks", "command_sha256", "heartbeat_path", "heartbeat_unix") + if any(key not in record["data"] for key in required): + raise TraceError(f"{phase} watchdog audit evidence is incomplete") + data = record["data"] + if not isinstance(data["pid"], int) or data["pid"] <= 1: + raise TraceError(f"{phase} watchdog audit PID is invalid") + if not isinstance(data["start_time_ticks"], int) or data["start_time_ticks"] <= 0: + raise TraceError(f"{phase} watchdog audit start time is invalid") + if not isinstance(data["command_sha256"], str) or re.fullmatch( + r"[0-9a-f]{64}", data["command_sha256"]) is None: + raise TraceError(f"{phase} watchdog audit command SHA-256 is invalid") + if not isinstance(data["heartbeat_path"], str) or not data["heartbeat_path"]: + raise TraceError(f"{phase} watchdog audit heartbeat path is invalid") + if not isinstance(data["heartbeat_unix"], int) or data["heartbeat_unix"] <= 0: + raise TraceError(f"{phase} watchdog audit heartbeat timestamp is invalid") + max_age = data.get("max_heartbeat_age_seconds") + if not isinstance(max_age, int) or max_age <= 0 or max_age > 30: + raise TraceError(f"{phase} watchdog audit heartbeat age is invalid") + if data["heartbeat_unix"] > record["created_unix"] or ( + record["created_unix"] - data["heartbeat_unix"] > max_age): + raise TraceError(f"{phase} watchdog audit heartbeat was stale when captured") def read_blob(self, event: dict[str, Any]) -> bytes: try: @@ -615,7 +658,7 @@ def _validate_component_schema(self, event: dict[str, Any]) -> None: "expert.weights": ("f32", config.get("experts_used")), "attn.source": ("i32", None), "attn.candidate_blocks": ("i32", None), - "attn.candidates": ("i32", config.get("index_top_k")), + "attn.candidates": ("i32", None), } if component not in widths: raise TraceError(f"unsupported trace component: {component}") @@ -626,6 +669,8 @@ def _validate_component_schema(self, event: dict[str, Any]) -> None: raise TraceError(f"{component} shape must be [{width},token_count]") if component == "attn.candidate_blocks" and shape[0] > config.get("candidate_topk_blocks", 0): raise TraceError("attn.candidate_blocks width exceeds candidate_topk_blocks") + if component in ("attn.source", "attn.candidates") and shape[0] > config.get("index_top_k", 0): + raise TraceError(f"{component} width exceeds index_top_k") if component == "expert.ids": values = struct.iter_unpack("= config.get("expert_count", 0) for value, in values): @@ -723,6 +768,7 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: None, "a trace contains duplicate phase/step/token/layer/component coordinates", ) + mismatches = [] all_keys = sorted(set(left_map) | set(right_map), key=event_order) for key in all_keys: left_event = left_map.get(key) @@ -730,7 +776,7 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: template = left_event or right_event assert template is not None if left_event is None or right_event is None: - return Mismatch( + mismatches.append(Mismatch( "artifact_missing", template["component"], template["phase"], @@ -738,10 +784,13 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: template["token_start"], template["layer"], "event is missing from " + ("left" if left_event is None else "right") + " trace", - ) + token_index=template["token_start"], + )) + continue + shape_mismatch = False for field in ("dtype", "shape", "token_count", "semantic_id_space"): if left_event.get(field) != right_event.get(field): - return Mismatch( + mismatches.append(Mismatch( "artifact_shape", template["component"], template["phase"], @@ -749,7 +798,12 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: template["token_start"], template["layer"], f"{field} differs: {left_event.get(field)!r} != {right_event.get(field)!r}", - ) + token_index=template["token_start"], + )) + shape_mismatch = True + break + if shape_mismatch: + continue if left_event["sha256"] == right_event["sha256"]: continue left_data = left.read_blob(left_event) @@ -772,7 +826,7 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: left_item = left_data[item_offset:item_offset + item_size] right_item = right_data[item_offset:item_offset + item_size] detail += f"; left=0x{left_item.hex()} right=0x{right_item.hex()}" - return Mismatch( + mismatches.append(Mismatch( classify(template["component"]), template["component"], template["phase"], @@ -784,8 +838,19 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: byte_offset=byte_offset, token_index=token_index, component_element_index=component_element_index, - ) - return None + )) + if not mismatches: + return None + phase_order = {"input": 0, "prefill": 1, "decode": 2, "metadata": -1} + component_order = {component: index for index, component in enumerate(HARD_FAILURE_COMPONENTS)} + return min(mismatches, key=lambda item: ( + phase_order.get(item.phase, 99), + item.token_index if item.token_index is not None else item.token_start, + item.step, + -1 if item.layer is None else item.layer, + component_order.get(item.component, 99), + item.component, + )) def report(left: TraceBundle, right: TraceBundle) -> dict[str, Any]: From 090b58125fc098e986d3992ac3cad93259faa230 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 21:14:44 -0700 Subject: [PATCH 10/56] deepseek41 : integrate trace producer contract Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-runtime.cpp | 31 ++++++++++ tests/test-deepseek41-trace.py | 50 +++++++++++++++- tools/deepseek-v41-trace/README.md | 4 +- tools/deepseek-v41-trace/llama-trace.cpp | 33 ++--------- tools/deepseek-v41-trace/run_ds4.py | 16 ++++-- tools/deepseek-v41-trace/trace-components.h | 64 +++++++++++++++++++++ tools/deepseek-v41-trace/trace_format.py | 37 +++++++----- 7 files changed, 188 insertions(+), 47 deletions(-) create mode 100644 tools/deepseek-v41-trace/trace-components.h diff --git a/tests/test-deepseek41-runtime.cpp b/tests/test-deepseek41-runtime.cpp index a03743a8551d..0ff9efe04b80 100644 --- a/tests/test-deepseek41-runtime.cpp +++ b/tests/test-deepseek41-runtime.cpp @@ -1,5 +1,6 @@ #include "../src/llama-dsv41.h" #include "../src/llama-arch.h" +#include "../tools/deepseek-v41-trace/trace-components.h" #include "ggml.h" @@ -308,14 +309,27 @@ static void test_graph_contract() { llama_dsv41_graph_trace_name("expert.ids", il) == "dsv41.trace.expert.ids.l" + std::to_string(il), "expert ID trace name is unstable"); + const auto expert_ids = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("expert.ids", il)); + check(expert_ids && expert_ids->component == "expert.ids" && + expert_ids->layer == (int) il && + std::string(expert_ids->semantic_id_space) == "original", + "exporter does not recognize original expert ID trace"); check( llama_dsv41_graph_trace_name("expert.weights", il) == "dsv41.trace.expert.weights.l" + std::to_string(il), "expert weight trace name is unstable"); + const auto expert_weights = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("expert.weights", il)); + check(expert_weights && expert_weights->component == "expert.weights" && + expert_weights->layer == (int) il, + "exporter does not recognize expert weight trace"); check( llama_dsv41_graph_trace_name("attn.source", il) == "dsv41.trace.attn.source.l" + std::to_string(il), "attention source trace name is unstable"); + const auto attention_source = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("attn.source", il)); + check(attention_source && attention_source->component == "attn.source" && + attention_source->layer == (int) il, + "exporter does not recognize attention source trace"); if (il > LLAMA_DSV41_CANDIDATE_SOURCE_LAYER && llama_dsv41_index_source_layer(il) == (int32_t) il) { candidate_trace_layers.push_back(il); @@ -323,6 +337,10 @@ static void test_graph_contract() { llama_dsv41_graph_trace_name("attn.candidates", il) == "dsv41.trace.attn.candidates.l" + std::to_string(il), "attention candidate trace name is unstable"); + const auto candidates = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("attn.candidates", il)); + check(candidates && candidates->component == "attn.candidates" && + candidates->layer == (int) il, + "exporter does not recognize propagated candidate trace"); } } check(ratio_count[0] == 2 && ratio_count[1] == 20 && ratio_count[2] == 18, @@ -336,12 +354,25 @@ static void test_graph_contract() { llama_dsv41_graph_trace_name("attn.candidate_blocks", 20) == "dsv41.trace.attn.candidate_blocks.l20", "candidate block trace name is unstable"); + const auto candidate_blocks = + dsv41_trace_parse_name(llama_dsv41_graph_trace_name("attn.candidate_blocks", 20)); + check(candidate_blocks && candidate_blocks->component == "attn.candidate_blocks" && + candidate_blocks->layer == 20, + "exporter does not recognize candidate block trace"); check( llama_dsv41_graph_trace_name("engram.row_ids", 1) == "dsv41.trace.engram.row_ids.l1" && llama_dsv41_graph_trace_name("engram.row_ids", 14) == "dsv41.trace.engram.row_ids.l14", "Engram row trace names are unstable"); + for (uint32_t layer : { 1u, 14u }) { + const auto engram = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("engram.row_ids", layer)); + check(engram && engram->component == "engram.row_ids" && + engram->layer == (int) layer, + "exporter does not recognize Engram row trace"); + } + check(!dsv41_trace_parse_name("dsv41.trace.attn.candidates.l20"), + "exporter accepted an unexpected candidate trace layer"); check(llama_dsv41_build_layer_plan(39, { 39 }, 1024).collapses_output, "final layer must preserve streams for carried-pre output collapse"); } diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 6d0a27cc0fb6..55339bb8c179 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -17,6 +17,7 @@ trace = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(trace) import run_llama +import run_ds4 import preflight @@ -541,7 +542,7 @@ def test_rejects_wrong_component_schema_and_same_bundle_compare(self) -> None: self.assertEqual(result["first_divergence"]["classification"], "artifact_identity") def test_rejects_empty_variable_width_components(self) -> None: - for component in ("attn.source", "attn.candidate_blocks"): + for component in ("attn.source", "attn.candidate_blocks", "attn.candidates"): with self.subTest(component=component), tempfile.TemporaryDirectory() as temp: writer = trace.TraceBundleWriter(Path(temp) / "trace", manifest()) with self.assertRaisesRegex(trace.TraceError, "nonzero"): @@ -558,6 +559,53 @@ def test_rejects_empty_variable_width_components(self) -> None: ) writer.events.close() + def test_rejects_zero_dimensions_globally(self) -> None: + with tempfile.TemporaryDirectory() as temp: + writer = trace.TraceBundleWriter(Path(temp) / "trace", manifest()) + with self.assertRaisesRegex(trace.TraceError, "nonzero"): + writer.add_event( + component="prompt.bytes", + phase="input", + step=0, + token_start=0, + token_count=1, + layer=None, + dtype="bytes", + shape=[0], + data=b"", + ) + writer.events.close() + + def test_rejects_dirty_ds4_checkout(self) -> None: + original = run_ds4.git_output + try: + run_ds4.git_output = lambda checkout, *args: ( + trace.DS4_REVISION if args == ("rev-parse", "HEAD") else " M runtime.py") + with self.assertRaisesRegex(preflight.PreflightError, "tracked or untracked"): + run_ds4.verify_checkout(Path("/tmp/ds4")) + finally: + run_ds4.git_output = original + + def test_rejects_weakened_coverage_contract(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + weakened = manifest() + del weakened["expected"]["components"]["expert.ids"]["decode"] + with trace.TraceBundleWriter(root, weakened) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "coverage contract"): + trace.TraceBundle(root) + + def test_rejects_prompt_token_count_not_bound_to_provenance(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + mismatched = manifest() + mismatched["expected"]["prompt_tokens"] = 1 + with trace.TraceBundleWriter(root, mismatched) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "prompt provenance"): + trace.TraceBundle(root) + def test_watchdog_stat_parser_handles_parentheses(self) -> None: fields = ["S", *[str(value) for value in range(4, 23)]] self.assertEqual(preflight.proc_start_time_ticks(f"123 (watch) dog) {' '.join(fields)}"), 22) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 5a93c509d870..8d68177848c1 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -12,7 +12,7 @@ Each trace is a directory: The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. -Internal tensors use raw ggml dimension order. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with width at most 512 and `token_count` in the second dimension, layer-20 candidate blocks as rank-2 i32 with width at most 2048, propagated candidates as rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. +Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with width at most 512 and `token_count` in the second dimension, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. Validate or compare bundles: @@ -40,7 +40,7 @@ Start at context 32768. `llama-deepseek-v41-prompt-builder` loads only the GGUF ## Strix execution gate -`run_ds4.py` verifies the pinned ds4 checkout and refuses model execution when swap is enabled, the watchdog lease or heartbeat is missing/stale, another matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. +`run_ds4.py` verifies that the pinned ds4 checkout has no tracked or untracked changes and refuses model execution when swap is enabled, the watchdog lease or heartbeat is missing/stale, another matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. The watchdog lease is JSON, not a bare PID: diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index ed70fbb033f7..8ebb2e2441a2 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -7,6 +7,7 @@ extern "C" { #include "hash/sha256/sha256.h" } #include "llama.h" +#include "trace-components.h" #include @@ -22,7 +23,6 @@ extern "C" { #include #include #include -#include #include #include #include @@ -32,7 +32,6 @@ namespace fs = std::filesystem; using json = nlohmann::ordered_json; static constexpr int TRACE_VERSION = 1; -static constexpr const char * TRACE_PREFIX = "dsv41.trace."; static std::string sha256_hex(const unsigned char digest[SHA256_DIGEST_SIZE]) { std::ostringstream stream; @@ -301,35 +300,15 @@ class trace_writer { void add_tensor(const ggml_tensor * tensor) { const std::string name = tensor->name; - std::string component; - const char * semantic_id_space = nullptr; - if (name.rfind("dsv41.trace.engram.row_ids.l", 0) == 0) { - component = "engram.row_ids"; - } else if (name.rfind("dsv41.trace.expert.ids.l", 0) == 0) { - component = "expert.ids"; - semantic_id_space = "original"; - } else if (name.rfind("dsv41.trace.expert.weights.l", 0) == 0) { - component = "expert.weights"; - } else if (name.rfind("dsv41.trace.attn.source.l", 0) == 0) { - component = "attn.source"; - } else if (name.rfind("dsv41.trace.attn.candidate_blocks.l", 0) == 0) { - component = "attn.candidate_blocks"; - } else if (name.rfind("dsv41.trace.attn.candidates.l", 0) == 0) { - component = "attn.candidates"; - } else { + const auto descriptor = dsv41_trace_parse_name(name); + if (!descriptor) { return; } - - static const std::regex layer_pattern(R"(\.l([0-9]+)$)"); - std::smatch match; - if (!std::regex_search(name, match, layer_pattern)) { - throw std::runtime_error("trace tensor name has no layer suffix: " + name); - } - const int layer = std::stoi(match[1].str()); const size_t size = ggml_nbytes(tensor); buffer.resize(size); ggml_backend_tensor_get(tensor, buffer.data(), 0, size); - add(component, layer, tensor_dtype(tensor), tensor_shape(tensor), buffer.data(), size, semantic_id_space); + add(descriptor->component, descriptor->layer, tensor_dtype(tensor), tensor_shape(tensor), + buffer.data(), size, descriptor->semantic_id_space); } bool has_error() const { @@ -381,7 +360,7 @@ class trace_writer { static bool trace_callback(ggml_tensor * tensor, bool ask, void * user_data) { auto * writer = static_cast(user_data); if (ask) { - return std::string(tensor->name).rfind(TRACE_PREFIX, 0) == 0; + return dsv41_trace_parse_name(tensor->name).has_value(); } try { writer->add_tensor(tensor); diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 4dc743cca3eb..d41c87c4b9bd 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -22,20 +22,28 @@ DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" -def read_revision(checkout: Path) -> str: +def git_output(checkout: Path, *args: str) -> str: try: return subprocess.check_output( - ["git", "-C", str(checkout), "rev-parse", "HEAD"], + ["git", "-C", str(checkout), *args], text=True, stderr=subprocess.STDOUT, ).strip() except (OSError, subprocess.CalledProcessError) as error: - raise PreflightError(f"cannot read ds4 revision: {error}") from error + raise PreflightError(f"ds4 git {' '.join(args)} failed: {error}") from error + + +def verify_checkout(checkout: Path) -> str: + revision = git_output(checkout, "rev-parse", "HEAD") + status = git_output(checkout, "status", "--porcelain", "--untracked-files=all") + if status: + raise PreflightError("ds4 checkout has tracked or untracked changes") + return revision def preflight(args: argparse.Namespace) -> dict[str, object]: checkout = resolved(args.checkout) - revision = read_revision(checkout) + revision = verify_checkout(checkout) if revision != DS4_REVISION: raise PreflightError(f"ds4 revision mismatch: expected {DS4_REVISION}, found {revision}") result = run_preflight( diff --git a/tools/deepseek-v41-trace/trace-components.h b/tools/deepseek-v41-trace/trace-components.h new file mode 100644 index 000000000000..b83e3df43a5c --- /dev/null +++ b/tools/deepseek-v41-trace/trace-components.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +struct dsv41_trace_descriptor { + std::string component; + int layer; + const char * semantic_id_space; +}; + +inline bool dsv41_trace_expected_layer(const std::string & component, int layer) { + if (component == "engram.row_ids") { + return layer == 1 || layer == 14; + } + if (component == "expert.ids" || component == "expert.weights" || component == "attn.source") { + return layer >= 0 && layer < 40; + } + if (component == "attn.candidate_blocks") { + return layer == 20; + } + if (component == "attn.candidates") { + return layer == 24 || layer == 28 || layer == 32 || layer == 36; + } + return false; +} + +inline std::optional dsv41_trace_parse_name(const std::string & name) { + struct prefix_entry { + const char * prefix; + const char * component; + const char * semantic_id_space; + }; + static const prefix_entry prefixes[] = { + {"dsv41.trace.engram.row_ids.l", "engram.row_ids", nullptr}, + {"dsv41.trace.expert.ids.l", "expert.ids", "original"}, + {"dsv41.trace.expert.weights.l", "expert.weights", nullptr}, + {"dsv41.trace.attn.source.l", "attn.source", nullptr}, + {"dsv41.trace.attn.candidate_blocks.l", "attn.candidate_blocks", nullptr}, + {"dsv41.trace.attn.candidates.l", "attn.candidates", nullptr}, + }; + for (const prefix_entry & entry : prefixes) { + const std::string prefix = entry.prefix; + if (name.rfind(prefix, 0) != 0 || name.size() == prefix.size()) { + continue; + } + int layer = 0; + for (size_t index = prefix.size(); index < name.size(); ++index) { + const char value = name[index]; + if (value < '0' || value > '9') { + return std::nullopt; + } + layer = 10*layer + value - '0'; + if (layer >= 40) { + return std::nullopt; + } + } + if (!dsv41_trace_expected_layer(entry.component, layer)) { + return std::nullopt; + } + return dsv41_trace_descriptor{entry.component, layer, entry.semantic_id_space}; + } + return std::nullopt; +} diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 2e10a69a0e05..594e000a4ded 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -51,13 +51,18 @@ "decode.greedy_token", ) -DEEPSEEK41_LAYERS = { - "engram.row_ids": [1, 14], - "expert.ids": list(range(40)), - "expert.weights": list(range(40)), - "attn.source": list(range(40)), - "attn.candidate_blocks": [20], - "attn.candidates": [24, 28, 32, 36], +DEEPSEEK41_EXPECTED_COMPONENTS = { + "prompt.bytes": {"layers": None, "input": "tokens"}, + "prompt.tokens": {"layers": None, "input": "tokens"}, + "engram.row_ids": {"layers": [1, 14], "prefill": "tokens", "decode": "steps"}, + "expert.ids": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "expert.weights": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.source": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.candidate_blocks": {"layers": [20], "prefill": "tokens", "decode": "steps"}, + "attn.candidates": {"layers": [24, 28, 32, 36], "prefill": "tokens", "decode": "steps"}, + "logits.prefill": {"layers": None, "prefill": "final"}, + "logits.decode": {"layers": None, "decode": "steps"}, + "decode.greedy_token": {"layers": None, "decode": "steps"}, } @@ -119,8 +124,8 @@ def canonical_json(data: Any) -> str: def element_count(shape: Iterable[int]) -> int: count = 1 for dim in shape: - if not isinstance(dim, int) or dim < 0: - raise TraceError(f"invalid shape dimension: {dim!r}") + if not isinstance(dim, int) or dim <= 0: + raise TraceError(f"shape dimension must be a nonzero positive integer: {dim!r}") count *= dim return count @@ -391,6 +396,11 @@ def _validate_manifest(self) -> None: for section in ("config", "comparison", "environment", "audits"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") + context = self.manifest["config"].get("context") + decode_steps = self.manifest["config"].get("decode_steps") + if not isinstance(context, int) or not isinstance(decode_steps, int) or ( + decode_steps <= 0 or context <= decode_steps): + raise TraceError("manifest context or decode_steps is invalid") if self.manifest["model"]["sha256"] != MODEL_SHA256: raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") if self.manifest["model"].get("architecture") != "deepseek41": @@ -416,7 +426,7 @@ def _validate_manifest(self) -> None: raise TraceError(f"cannot read prompt provenance: {error}") from error if sha256_bytes(provenance_bytes) != provenance_sha256: raise TraceError("prompt provenance SHA-256 mismatch") - expected_target = self.manifest["config"].get("context", 0) - self.manifest["config"].get("decode_steps", 0) + expected_target = context - decode_steps provenance_checks = { "format": "dsv41-prompt-provenance", "version": 1, @@ -552,14 +562,15 @@ def _validate_coverage(self) -> None: raise TraceError("expected decode_steps is invalid") if self.manifest.get("config", {}).get("decode_steps") != decode_steps: raise TraceError("expected decode_steps does not match config") + if prompt_tokens != self.manifest.get("prompt", {}).get("target_tokens"): + raise TraceError("expected prompt_tokens does not match prompt provenance") for event in self.events: self._validate_component_schema(event) if not isinstance(components, dict): raise TraceError("expected components are invalid") if self.manifest.get("model", {}).get("architecture") == "deepseek41": - for component, layers in DEEPSEEK41_LAYERS.items(): - if components.get(component, {}).get("layers") != layers: - raise TraceError(f"DeepSeek V4.1 expected layers are invalid for {component}") + if components != DEEPSEEK41_EXPECTED_COMPONENTS: + raise TraceError("DeepSeek V4.1 expected component coverage contract is invalid") by_component: dict[str, list[dict[str, Any]]] = {} for event in self.events: From 7ae899eda76494b967a8d0e594355bc47a2fd1e8 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 21:20:39 -0700 Subject: [PATCH 11/56] deepseek41 : reject malformed trace names Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-runtime.cpp | 17 ++++++++++++++++- tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/llama-trace.cpp | 6 +++--- tools/deepseek-v41-trace/trace-components.h | 14 +++++++++++--- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/test-deepseek41-runtime.cpp b/tests/test-deepseek41-runtime.cpp index 0d09ac9d461f..87fb8afb633b 100644 --- a/tests/test-deepseek41-runtime.cpp +++ b/tests/test-deepseek41-runtime.cpp @@ -378,8 +378,23 @@ static void test_graph_contract() { engram->layer == (int) layer, "exporter does not recognize Engram row trace"); } - check(!dsv41_trace_parse_name("dsv41.trace.attn.candidates.l20"), + expect_throw( + [] { + dsv41_trace_select_name("dsv41.trace.attn.candidates.l20"); + }, "exporter accepted an unexpected candidate trace layer"); + expect_throw( + [] { + dsv41_trace_select_name("dsv41.trace.attn.candidates.layer24"); + }, + "exporter accepted a malformed trace layer suffix"); + expect_throw( + [] { + dsv41_trace_select_name("dsv41.trace.unknown.l24"); + }, + "exporter accepted an unknown reserved trace tensor name"); + check(!dsv41_trace_select_name("dsv41_attn_candidates_l24"), + "exporter treated an ordinary graph tensor as reserved"); check(llama_dsv41_build_layer_plan(39, { 39 }, 1024).collapses_output, "final layer must preserve streams for carried-pre output collapse"); } diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 8d68177848c1..ddadb578308a 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -10,7 +10,7 @@ Each trace is a directory: - `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. - `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. -The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. +The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with width at most 512 and `token_count` in the second dimension, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 8ebb2e2441a2..7aaa219890b4 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -359,10 +359,10 @@ class trace_writer { static bool trace_callback(ggml_tensor * tensor, bool ask, void * user_data) { auto * writer = static_cast(user_data); - if (ask) { - return dsv41_trace_parse_name(tensor->name).has_value(); - } try { + if (ask) { + return dsv41_trace_select_name(tensor->name); + } writer->add_tensor(tensor); return !writer->has_error(); } catch (const std::exception & error) { diff --git a/tools/deepseek-v41-trace/trace-components.h b/tools/deepseek-v41-trace/trace-components.h index b83e3df43a5c..4e88f269288c 100644 --- a/tools/deepseek-v41-trace/trace-components.h +++ b/tools/deepseek-v41-trace/trace-components.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include struct dsv41_trace_descriptor { @@ -48,17 +49,24 @@ inline std::optional dsv41_trace_parse_name(const std::s for (size_t index = prefix.size(); index < name.size(); ++index) { const char value = name[index]; if (value < '0' || value > '9') { - return std::nullopt; + throw std::runtime_error("malformed reserved trace tensor name: " + name); } layer = 10*layer + value - '0'; if (layer >= 40) { - return std::nullopt; + throw std::runtime_error("unexpected reserved trace tensor layer: " + name); } } if (!dsv41_trace_expected_layer(entry.component, layer)) { - return std::nullopt; + throw std::runtime_error("unexpected reserved trace tensor layer: " + name); } return dsv41_trace_descriptor{entry.component, layer, entry.semantic_id_space}; } + if (name.rfind("dsv41.trace.", 0) == 0) { + throw std::runtime_error("malformed reserved trace tensor name: " + name); + } return std::nullopt; } + +inline bool dsv41_trace_select_name(const std::string & name) { + return dsv41_trace_parse_name(name).has_value(); +} From 8acdb46f83251b531b896658d5c913ba8edecf4a Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 21:36:56 -0700 Subject: [PATCH 12/56] scripts : forward SIGHUP through memory watchdog Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 2 +- scripts/strix_memory_watchdog.py | 2 +- tests/test_strix_memory_watchdog.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index ccd04f7ab667..0ba9e2e7721e 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -14,7 +14,7 @@ The wrapper performs these checks and actions: - It sends `SIGKILL` at 118 GiB used or 30 seconds after `SIGTERM`. - It reports `grace_timeout` if any descendant requires `SIGKILL` after the soft-threshold grace period, even when the direct child exited earlier. - It sends `SIGKILL` and fails if swap appears or required procfs data becomes unavailable during execution. -- It forwards wrapper `SIGINT` or `SIGTERM` to the process group, waits the configured grace period, then sends `SIGKILL` if any group member remains. +- It forwards wrapper `SIGHUP`, `SIGINT`, or `SIGTERM` to the process group, waits the configured grace period, then sends `SIGKILL` if any group member remains. - It checks the process group after the direct child exits and cleans up remaining descendants before returning the child's classification. - It applies the same bounded process-group cleanup if an unexpected post-launch error occurs. - It propagates an unmonitored child exit code. A signal exit uses the shell convention `128 + signal`. diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index 00a9fab78d81..a17c76dc960b 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -36,7 +36,7 @@ MEMINFO_VALUE_RE = re.compile(r"([0-9]+) kB") SWAPS_HEADER = ["Filename", "Type", "Size", "Used", "Priority"] -PARENT_SIGNALS = (signal.SIGINT, signal.SIGTERM) +PARENT_SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM) class ProcfsError(RuntimeError): diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index 898298efb9b9..eca818da0f6f 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -228,6 +228,7 @@ def _write_procfs_fixture(root: Path) -> None: def test_parent_signals_leave_no_child_or_grandchild(self) -> None: child_code = ( "import os,signal,sys,time;" + "signal.signal(signal.SIGHUP,signal.SIG_IGN);" "signal.signal(signal.SIGINT,signal.SIG_IGN);" "signal.signal(signal.SIGTERM,signal.SIG_IGN);" "grandchild=os.fork();" @@ -238,7 +239,11 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: "f'{os.getpid()} {grandchild}\\n');" " time.sleep(30)\n" ) - for signal_number in (signal.SIGINT, signal.SIGTERM): + for signal_number in ( + signal.SIGHUP, + signal.SIGINT, + signal.SIGTERM, + ): with self.subTest(signal=signal.Signals(signal_number).name): with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) From 59833018814c0883f995848912cd5a52889c5303 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 22:37:06 -0700 Subject: [PATCH 13/56] scripts : publish watchdog-owned validation lease Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 43 ++ scripts/strix_memory_watchdog.py | 914 ++++++++++++++++++++++++++-- tests/test_strix_memory_watchdog.py | 475 ++++++++++++++- 3 files changed, 1385 insertions(+), 47 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 0ba9e2e7721e..5565716a0a3e 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -25,6 +25,48 @@ Use `--procfs-root` to select a different procfs mount or a test fixture. `--sof The wrapper writes timestamped JSON Lines records to standard error. Preflight, sample, signal, and final records include total, available, used, and peak-used bytes, swap entry count, child status, process-group status, threshold reason, and final classification where applicable. Signal records are written immediately after each process-group signal. Child standard input, standard output, and standard error are inherited unchanged. +## Watchdog-owned validation lease + +Use all three artifact options together when another process must prove that it is inside the active watchdog process group: + +```sh +./scripts/strix_memory_watchdog.py \ + --lease-path /run/deepseek-v41/watchdog-lease.json \ + --heartbeat-path /run/deepseek-v41/watchdog-heartbeat.json \ + --audit-path /run/deepseek-v41/watchdog-audit.jsonl \ + -- \ + python3 tools/deepseek-v41-trace/run_matrix.py +``` + +The watchdog creates the persistent audit before launch, then atomically creates the lease and heartbeat after `Popen` returns. Existing artifact paths are rejected rather than overwritten. The child receives the resolved paths through `STRIX_MEMORY_WATCHDOG_LEASE_PATH`, `STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH`, and `STRIX_MEMORY_WATCHDOG_AUDIT_PATH`. It also receives `STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS`. + +The child can run before the first atomic lease rename. A matching preflight must retry the inherited lease path for a bounded interval and fail closed if a complete valid lease does not appear. It must not accept a lease path supplied separately by the operator. + +Lease format `strix-memory-watchdog-lease`, version 1, contains: + +- `lease_id` and active/final `state` +- `watchdog_pid`, `watchdog_start_time_utc`, Linux `watchdog_start_time_ticks`, `watchdog_command_sha256`, `watchdog_script_path`, and `watchdog_script_sha256` +- exact `soft_bytes`, `emergency_bytes`, and `strict_ceiling_bytes` +- `procfs_root` +- `child_pid`, `child_process_group_id`, `command`, and `child_command_sha256` +- `heartbeat_path`, `max_heartbeat_age_seconds`, and `audit_path` +- the authoritative `final` audit record after termination + +Heartbeat format `strix-memory-watchdog-heartbeat`, version 1, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample atomically replaces the heartbeat and includes the complete sample audit record. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. + +A matching Linux preflight must verify all of the following: + +- The inherited lease, heartbeat, and audit paths match the paths inside the lease. +- The expected repository script path hashes to `watchdog_script_sha256`. +- `/proc//stat` start ticks and `/proc//cmdline` SHA-256 match the lease. +- The watchdog command line names the expected script. +- The current process group equals `child_process_group_id`, whose leader is `child_pid` and whose parent is `watchdog_pid`. +- The command identity is expected, the procfs root is `/proc`, and thresholds are exactly 116 GiB soft, 118 GiB emergency, and 120 GiB strict ceiling for the final run. +- The heartbeat identity matches the lease and its monotonic timestamp is not older than `max_heartbeat_age_seconds`. +- The persistent audit exists and contains watchdog JSONL records. + +These checks bind the validation process to the live canonical watchdog. A standalone heartbeat helper has a different PID, start time, command line, script hash, and process group and cannot satisfy the lease. + Exit classifications are authoritative in the final JSON record. Operational failures use these exit codes: | Exit code | Classification | @@ -35,6 +77,7 @@ Exit classifications are authoritative in the final JSON record. Operational fai | 5 | emergency threshold reached | | 6 | soft-threshold grace period expired | | 7 | process-group signaling or termination failure | +| 8 | lease, heartbeat, or persistent audit failure | | 70 | unexpected post-launch error | | 127 | command launch failure | diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index a17c76dc960b..0bb11a0622c3 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -3,10 +3,12 @@ from __future__ import annotations import argparse +import hashlib import json import math import os import re +import secrets import signal import subprocess import sys @@ -15,7 +17,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import IO, Protocol +from typing import IO, Any, Protocol GIB = 1024**3 @@ -24,6 +26,12 @@ DEFAULT_EMERGENCY_BYTES = 118 * GIB DEFAULT_GRACE_SECONDS = 30.0 DEFAULT_SAMPLE_INTERVAL_SECONDS = 1.0 +DEFAULT_HEARTBEAT_MAX_AGE_SECONDS = 5.0 + +LEASE_FORMAT = "strix-memory-watchdog-lease" +LEASE_VERSION = 1 +HEARTBEAT_FORMAT = "strix-memory-watchdog-heartbeat" +HEARTBEAT_VERSION = 1 EXIT_PROCFS_ERROR = 2 EXIT_SWAP_ACTIVE = 3 @@ -31,6 +39,7 @@ EXIT_EMERGENCY_LIMIT = 5 EXIT_GRACE_TIMEOUT = 6 EXIT_SIGNAL_ERROR = 7 +EXIT_LEASE_ERROR = 8 EXIT_INTERNAL_ERROR = 70 EXIT_LAUNCH_ERROR = 127 @@ -47,6 +56,16 @@ class ProcessGroupError(RuntimeError): pass +class ArtifactError(RuntimeError): + def __init__(self, component: str, detail: str): + self.component = component + super().__init__(detail) + + +class LeaseValidationError(RuntimeError): + pass + + class ParentSignal(RuntimeError): def __init__(self, signal_number: int): self.signal_number = signal_number @@ -80,6 +99,13 @@ class RuntimeState: peak_used_bytes: int +@dataclass(frozen=True) +class ArtifactPaths: + lease: Path + heartbeat: Path + audit: Path + + @dataclass(frozen=True) class WatchdogConfig: command: tuple[str, ...] @@ -88,8 +114,16 @@ class WatchdogConfig: emergency_bytes: int = DEFAULT_EMERGENCY_BYTES grace_seconds: float = DEFAULT_GRACE_SECONDS sample_interval_seconds: float = DEFAULT_SAMPLE_INTERVAL_SECONDS + lease_path: Path | None = None + heartbeat_path: Path | None = None + audit_path: Path | None = None + heartbeat_max_age_seconds: float = DEFAULT_HEARTBEAT_MAX_AGE_SECONDS + + @property + def lease_enabled(self) -> bool: + return self.lease_path is not None - def validate(self) -> None: + def validate(self) -> ArtifactPaths | None: if not self.command: raise ValueError("a command is required after --") if self.soft_bytes <= 0: @@ -105,6 +139,45 @@ def validate(self) -> None: or self.sample_interval_seconds <= 0 ): raise ValueError("sample interval must be greater than zero") + lease_paths = ( + self.lease_path, + self.heartbeat_path, + self.audit_path, + ) + if any(path is not None for path in lease_paths) and not all( + path is not None for path in lease_paths + ): + raise ValueError( + "lease, heartbeat, and audit paths must be specified together" + ) + if self.lease_enabled: + assert self.lease_path is not None + assert self.heartbeat_path is not None + assert self.audit_path is not None + try: + paths = ArtifactPaths( + self.lease_path.expanduser().resolve(), + self.heartbeat_path.expanduser().resolve(), + self.audit_path.expanduser().resolve(), + ) + except (OSError, RuntimeError) as exc: + raise ValueError( + f"cannot resolve watchdog artifact path: {exc}" + ) from exc + if len({paths.lease, paths.heartbeat, paths.audit}) != 3: + raise ValueError( + "lease, heartbeat, and audit paths must be distinct" + ) + if ( + not math.isfinite(self.heartbeat_max_age_seconds) + or self.heartbeat_max_age_seconds + <= self.sample_interval_seconds + ): + raise ValueError( + "heartbeat max age must be greater than sample interval" + ) + return paths + return None class ProcfsReader: @@ -171,6 +244,532 @@ def _parse_swaps(content: str) -> tuple[str, ...]: return tuple(entries) +def _timestamp_utc( + wall_clock: Callable[[], datetime] | None = None, +) -> str: + timestamp = (wall_clock or ( + lambda: datetime.now(timezone.utc) + ))().astimezone(timezone.utc) + return timestamp.isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ) + + +def _sha256_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def _sha256_file(path: Path) -> str: + try: + return _sha256_bytes(path.read_bytes()) + except OSError as exc: + detail = exc.strerror or str(exc) + raise ArtifactError( + "lease", f"cannot hash {path}: {detail}" + ) from exc + + +def _command_sha256(command: Sequence[str]) -> str: + encoded = json.dumps( + list(command), + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + return _sha256_bytes(encoded) + + +def _read_proc_bytes(root: Path, process_id: int, name: str) -> bytes: + path = root / str(process_id) / name + try: + return path.read_bytes() + except OSError as exc: + detail = exc.strerror or str(exc) + raise LeaseValidationError( + f"cannot read {path}: {detail}" + ) from exc + + +def _parse_proc_stat(content: str) -> tuple[int, int, int]: + close_paren = content.rfind(")") + if close_paren < 0: + raise LeaseValidationError("malformed process stat") + fields = content[close_paren + 1:].split() + if len(fields) < 20: + raise LeaseValidationError("malformed process stat") + try: + return int(fields[1]), int(fields[2]), int(fields[19]) + except ValueError as exc: + raise LeaseValidationError("malformed process stat") from exc + + +def _read_proc_stat( + root: Path, process_id: int +) -> tuple[int, int, int]: + content = _read_proc_bytes( + root, process_id, "stat" + ).decode("utf-8") + return _parse_proc_stat(content) + + +def _write_json_atomic( + path: Path, + value: dict[str, object], + *, + create: bool = False, +) -> None: + parent = path.parent + temp_path = parent / ( + f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp" + ) + payload = ( + json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode("utf-8") + try: + descriptor = os.open( + temp_path, + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o600, + ) + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + if create: + os.link(temp_path, path) + temp_path.unlink() + else: + os.replace(temp_path, path) + directory_descriptor = os.open(parent, os.O_RDONLY) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + except OSError as exc: + try: + temp_path.unlink() + except FileNotFoundError: + pass + detail = exc.strerror or str(exc) + action = "create" if create else "write" + raise ArtifactError( + "lease", f"cannot atomically {action} {path}: {detail}" + ) from exc + + +class LeaseManager: + def __init__( + self, + config: WatchdogConfig, + paths: ArtifactPaths, + *, + process_procfs_root: Path = Path("/proc"), + wall_clock: Callable[[], datetime] | None = None, + monotonic_ns: Callable[[], int] | None = None, + ): + self.config = config + self.lease_path = paths.lease + self.heartbeat_path = paths.heartbeat + self.audit_path = paths.audit + self.process_procfs_root = process_procfs_root + self.wall_clock = wall_clock + self.monotonic_ns = monotonic_ns or time.monotonic_ns + self.lease_id = secrets.token_hex(16) + self.sequence = 0 + self.lease: dict[str, object] | None = None + + def _watchdog_identity(self) -> dict[str, object]: + script_path = Path(__file__).resolve() + cmdline_path = ( + self.process_procfs_root / str(os.getpid()) / "cmdline" + ) + proc_start_time_ticks: int | None = None + try: + cmdline = cmdline_path.read_bytes() + _, _, proc_start_time_ticks = _read_proc_stat( + self.process_procfs_root, os.getpid() + ) + except (OSError, LeaseValidationError): + if sys.platform.startswith("linux"): + raise ArtifactError( + "lease", + "cannot read watchdog process identity from procfs", + ) + cmdline = b"\0".join( + os.fsencode(argument) for argument in sys.argv + ) + return { + "pid": os.getpid(), + "start_time_utc": _timestamp_utc(self.wall_clock), + "proc_start_time_ticks": proc_start_time_ticks, + "cmdline_sha256": _sha256_bytes(cmdline), + "script_path": str(script_path), + "script_sha256": _sha256_file(script_path), + } + + def _heartbeat_record( + self, + state: str, + sample: dict[str, object] | None = None, + ) -> dict[str, object]: + assert self.lease is not None + self.sequence += 1 + record: dict[str, object] = { + "format": HEARTBEAT_FORMAT, + "version": HEARTBEAT_VERSION, + "lease_id": self.lease_id, + "sequence": self.sequence, + "state": state, + "updated_at": _timestamp_utc(self.wall_clock), + "updated_monotonic_ns": self.monotonic_ns(), + "watchdog_pid": self.lease["watchdog_pid"], + "watchdog_start_time_ticks": ( + self.lease["watchdog_start_time_ticks"] + ), + "child_pid": self.lease["child_pid"], + "child_process_group_id": ( + self.lease["child_process_group_id"] + ), + } + if sample is not None: + record["sample"] = sample + return record + + def start(self, child: ProcessHandle) -> None: + watchdog_identity = self._watchdog_identity() + self.lease = { + "format": LEASE_FORMAT, + "version": LEASE_VERSION, + "lease_id": self.lease_id, + "state": "active", + "watchdog_pid": watchdog_identity["pid"], + "watchdog_start_time_utc": ( + watchdog_identity["start_time_utc"] + ), + "watchdog_start_time_ticks": ( + watchdog_identity["proc_start_time_ticks"] + ), + "watchdog_command_sha256": ( + watchdog_identity["cmdline_sha256"] + ), + "watchdog_script_path": watchdog_identity["script_path"], + "watchdog_script_sha256": ( + watchdog_identity["script_sha256"] + ), + "soft_bytes": self.config.soft_bytes, + "emergency_bytes": self.config.emergency_bytes, + "strict_ceiling_bytes": STRICT_CEILING_BYTES, + "child_pid": child.pid, + "child_process_group_id": child.pid, + "command": list(self.config.command), + "child_command_sha256": _command_sha256( + self.config.command + ), + "heartbeat_path": str(self.heartbeat_path), + "max_heartbeat_age_seconds": ( + self.config.heartbeat_max_age_seconds + ), + "audit_path": str(self.audit_path), + "procfs_root": str( + self.config.procfs_root.expanduser().resolve() + ), + } + heartbeat = self._heartbeat_record("active") + _write_json_atomic(self.heartbeat_path, heartbeat, create=True) + _write_json_atomic(self.lease_path, self.lease, create=True) + + def update_heartbeat(self, sample: dict[str, object]) -> None: + heartbeat = self._heartbeat_record("active", sample) + _write_json_atomic(self.heartbeat_path, heartbeat) + + def finalize(self, final_record: dict[str, object]) -> None: + if self.lease is None: + return + self.lease["state"] = "final" + self.lease["final"] = final_record + heartbeat = self._heartbeat_record("final") + _write_json_atomic(self.heartbeat_path, heartbeat) + _write_json_atomic(self.lease_path, self.lease) + + +def _read_json_object(path: Path) -> dict[str, object]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise LeaseValidationError( + f"cannot read valid JSON from {path}: {exc}" + ) from exc + if not isinstance(value, dict): + raise LeaseValidationError(f"{path} must contain a JSON object") + return value + + +def _require_int(value: object, field: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise LeaseValidationError(f"lease field {field} is invalid") + return value + + +def _require_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value: + raise LeaseValidationError(f"lease field {field} is invalid") + return value + + +def validate_active_lease( + lease_path: Path, + *, + expected_script_path: Path, + expected_soft_bytes: int = DEFAULT_SOFT_BYTES, + expected_emergency_bytes: int = DEFAULT_EMERGENCY_BYTES, + expected_procfs_root: Path = Path("/proc"), + expected_command_sha256: str | None = None, + expected_heartbeat_path: Path | None = None, + expected_audit_path: Path | None = None, + expected_max_heartbeat_age_seconds: float | None = None, + current_process_id: int | None = None, + process_procfs_root: Path = Path("/proc"), + monotonic_ns: Callable[[], int] | None = None, +) -> dict[str, object]: + lease_path = lease_path.expanduser().resolve() + lease = _read_json_object(lease_path) + if ( + lease.get("format") != LEASE_FORMAT + or lease.get("version") != LEASE_VERSION + or lease.get("state") != "active" + ): + raise LeaseValidationError("lease format, version, or state is invalid") + + script_path = Path( + _require_string( + lease.get("watchdog_script_path"), + "watchdog_script_path", + ) + ).resolve() + expected_script_path = expected_script_path.expanduser().resolve() + if script_path != expected_script_path: + raise LeaseValidationError("watchdog script path does not match") + script_sha256 = _require_string( + lease.get("watchdog_script_sha256"), + "watchdog_script_sha256", + ) + if script_sha256 != _sha256_file(expected_script_path): + raise LeaseValidationError("watchdog script SHA does not match") + + if ( + _require_int( + lease.get("soft_bytes"), "soft_bytes" + ) + != expected_soft_bytes + or _require_int( + lease.get("emergency_bytes"), + "emergency_bytes", + ) + != expected_emergency_bytes + or _require_int( + lease.get("strict_ceiling_bytes"), + "strict_ceiling_bytes", + ) + != STRICT_CEILING_BYTES + ): + raise LeaseValidationError("watchdog thresholds do not match") + lease_procfs_root = Path( + _require_string(lease.get("procfs_root"), "procfs_root") + ).resolve() + if lease_procfs_root != expected_procfs_root.expanduser().resolve(): + raise LeaseValidationError("watchdog procfs root does not match") + + watchdog_pid = _require_int( + lease.get("watchdog_pid"), "watchdog_pid" + ) + watchdog_start_ticks = _require_int( + lease.get("watchdog_start_time_ticks"), + "watchdog_start_time_ticks", + ) + _, _, live_watchdog_start_ticks = _read_proc_stat( + process_procfs_root, watchdog_pid + ) + if live_watchdog_start_ticks != watchdog_start_ticks: + raise LeaseValidationError("watchdog process start time does not match") + live_cmdline = _read_proc_bytes( + process_procfs_root, watchdog_pid, "cmdline" + ) + if _sha256_bytes(live_cmdline) != _require_string( + lease.get("watchdog_command_sha256"), + "watchdog_command_sha256", + ): + raise LeaseValidationError("watchdog command line does not match") + script_named = False + watchdog_cwd: Path | None = None + for raw_argument in live_cmdline.split(b"\0"): + if not raw_argument: + continue + argument_path = Path(os.fsdecode(raw_argument)).expanduser() + if not argument_path.is_absolute(): + if watchdog_cwd is None: + try: + watchdog_cwd = ( + process_procfs_root + / str(watchdog_pid) + / "cwd" + ).resolve() + except OSError as exc: + raise LeaseValidationError( + "cannot resolve watchdog working directory" + ) from exc + argument_path = watchdog_cwd / argument_path + if argument_path.resolve() == expected_script_path: + script_named = True + break + if not script_named: + raise LeaseValidationError( + "watchdog command line does not name the expected script" + ) + + child_pid = _require_int(lease.get("child_pid"), "child_pid") + process_group_id = _require_int( + lease.get("child_process_group_id"), + "child_process_group_id", + ) + child_parent_pid, child_group_id, _ = _read_proc_stat( + process_procfs_root, child_pid + ) + if ( + child_parent_pid != watchdog_pid + or child_group_id != process_group_id + or process_group_id != child_pid + ): + raise LeaseValidationError( + "monitored child parent or process group does not match" + ) + command = lease.get("command") + if ( + not isinstance(command, list) + or not command + or not all(isinstance(argument, str) for argument in command) + ): + raise LeaseValidationError("lease field command is invalid") + command_sha256 = _require_string( + lease.get("child_command_sha256"), + "child_command_sha256", + ) + if command_sha256 != _command_sha256(command): + raise LeaseValidationError("monitored command SHA is invalid") + if ( + expected_command_sha256 is not None + and command_sha256 != expected_command_sha256 + ): + raise LeaseValidationError("monitored command SHA does not match") + + process_id = ( + current_process_id + if current_process_id is not None + else os.getpid() + ) + _, current_group_id, _ = _read_proc_stat( + process_procfs_root, process_id + ) + if current_group_id != process_group_id: + raise LeaseValidationError( + "current process is outside the monitored process group" + ) + + heartbeat_path = Path( + _require_string( + lease.get("heartbeat_path"), "heartbeat_path" + ) + ).resolve() + if ( + expected_heartbeat_path is not None + and heartbeat_path + != expected_heartbeat_path.expanduser().resolve() + ): + raise LeaseValidationError("heartbeat path does not match") + heartbeat_max_age = lease.get("max_heartbeat_age_seconds") + if ( + not isinstance(heartbeat_max_age, (int, float)) + or isinstance(heartbeat_max_age, bool) + or not math.isfinite(heartbeat_max_age) + or heartbeat_max_age <= 0 + ): + raise LeaseValidationError( + "lease field max_heartbeat_age_seconds is invalid" + ) + if ( + expected_max_heartbeat_age_seconds is not None + and heartbeat_max_age != expected_max_heartbeat_age_seconds + ): + raise LeaseValidationError("heartbeat max age does not match") + heartbeat = _read_json_object(heartbeat_path) + lease_id = _require_string(lease.get("lease_id"), "lease_id") + if ( + heartbeat.get("format") != HEARTBEAT_FORMAT + or heartbeat.get("version") != HEARTBEAT_VERSION + or heartbeat.get("state") != "active" + or heartbeat.get("lease_id") != lease_id + or heartbeat.get("watchdog_pid") != watchdog_pid + or heartbeat.get("watchdog_start_time_ticks") + != watchdog_start_ticks + or heartbeat.get("child_pid") != child_pid + or heartbeat.get("child_process_group_id") != process_group_id + ): + raise LeaseValidationError("heartbeat identity does not match lease") + updated_monotonic_ns = _require_int( + heartbeat.get("updated_monotonic_ns"), + "heartbeat.updated_monotonic_ns", + ) + _require_int(heartbeat.get("sequence"), "heartbeat.sequence") + _require_string(heartbeat.get("updated_at"), "heartbeat.updated_at") + now_monotonic_ns = (monotonic_ns or time.monotonic_ns)() + age_ns = now_monotonic_ns - updated_monotonic_ns + if age_ns < 0 or age_ns > int(heartbeat_max_age * 1_000_000_000): + raise LeaseValidationError("watchdog heartbeat is stale") + + audit_path = Path( + _require_string(lease.get("audit_path"), "audit_path") + ).resolve() + if ( + expected_audit_path is not None + and audit_path != expected_audit_path.expanduser().resolve() + ): + raise LeaseValidationError("persistent audit path does not match") + try: + first_line = next( + line + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + if line + ) + first_record = json.loads(first_line) + if ( + not isinstance(first_record, dict) + or not isinstance(first_record.get("event"), str) + or not isinstance(first_record.get("timestamp"), str) + ): + raise LeaseValidationError( + "persistent audit does not contain watchdog records" + ) + except StopIteration as exc: + raise LeaseValidationError("persistent audit is empty") from exc + except (UnicodeError, json.JSONDecodeError) as exc: + raise LeaseValidationError( + "persistent audit does not contain valid JSONL" + ) from exc + except LeaseValidationError: + raise + except OSError as exc: + raise LeaseValidationError( + f"cannot inspect persistent audit {audit_path}: {exc}" + ) from exc + return lease + + class AuditLogger: def __init__( self, @@ -178,23 +777,77 @@ def __init__( wall_clock: Callable[[], datetime] | None = None, ): self.stream = stream - self.wall_clock = wall_clock or ( - lambda: datetime.now(timezone.utc) - ) - - def emit(self, event: str, **fields: object) -> None: - timestamp = self.wall_clock().astimezone(timezone.utc) + self.wall_clock = wall_clock + self.persistent_stream: IO[str] | None = None + self.lease_manager: LeaseManager | None = None + self.finalized = False + self.final_exit_code = EXIT_INTERNAL_ERROR + + def open_persistent(self, path: Path) -> None: + resolved_path = path.expanduser().resolve() + try: + descriptor = os.open( + resolved_path, + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o600, + ) + self.persistent_stream = os.fdopen( + descriptor, "w", encoding="utf-8" + ) + except OSError as exc: + detail = exc.strerror or str(exc) + raise ArtifactError( + "audit", + f"cannot create persistent audit {resolved_path}: {detail}", + ) from exc + + def close(self) -> None: + if self.persistent_stream is not None: + self.persistent_stream.close() + self.persistent_stream = None + + def disable_component(self, component: str) -> None: + if component == "audit": + self.close() + elif component == "lease": + self.lease_manager = None + + def emit(self, event: str, **fields: object) -> dict[str, object]: record = { - "timestamp": timestamp.isoformat(timespec="milliseconds").replace( - "+00:00", "Z" - ), + "timestamp": _timestamp_utc(self.wall_clock), "event": event, **fields, } - self.stream.write( - json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" + line = ( + json.dumps(record, sort_keys=True, separators=(",", ":")) + + "\n" ) + self.stream.write(line) self.stream.flush() + if self.persistent_stream is not None: + try: + self.persistent_stream.write(line) + self.persistent_stream.flush() + os.fsync(self.persistent_stream.fileno()) + except OSError as exc: + detail = exc.strerror or str(exc) + raise ArtifactError( + "audit", + f"cannot write persistent audit: {detail}", + ) from exc + return record + + def heartbeat(self, sample: dict[str, object]) -> None: + if self.lease_manager is not None: + self.lease_manager.update_heartbeat(sample) + + def finalize(self, record: dict[str, object]) -> None: + if self.lease_manager is not None: + self.lease_manager.finalize(record) + + def mark_final(self, exit_code: int) -> None: + self.finalized = True + self.final_exit_code = exit_code def _child_status(returncode: int | None, started: bool = True) -> str: @@ -253,8 +906,35 @@ def _emit_final( fields.update(classification=classification, exit_code=exit_code) if error: fields["error"] = error - audit.emit("final", **fields) - return exit_code + previous_mask = signal.pthread_sigmask( + signal.SIG_BLOCK, PARENT_SIGNALS + ) + try: + try: + record = audit.emit("final", **fields) + audit.finalize(record) + except ArtifactError as exc: + audit.disable_component(exc.component) + fields.update( + classification="lease_error", + exit_code=EXIT_LEASE_ERROR, + threshold_reason="watchdog artifact finalization failed", + error=f"{exc.component}: {exc}", + ) + try: + record = audit.emit("final", **fields) + except ArtifactError as nested_exc: + audit.disable_component(nested_exc.component) + record = audit.emit("final", **fields) + try: + audit.finalize(record) + except ArtifactError as nested_exc: + audit.disable_component(nested_exc.component) + exit_code = EXIT_LEASE_ERROR + audit.mark_final(exit_code) + return exit_code + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) def _signal_process_group(process_group_id: int, signal_number: int) -> str: @@ -276,6 +956,8 @@ def _process_group_alive(process_group_id: int) -> bool: os.killpg(process_group_id, 0) except ProcessLookupError: return False + except PermissionError: + return True except OSError as exc: detail = exc.strerror or str(exc) raise ProcessGroupError( @@ -289,16 +971,16 @@ def _raise_parent_signal(signal_number: int, _frame: object) -> None: def _set_parent_signal_handlers( - handler: signal.Handlers, -) -> dict[int, signal.Handlers]: - previous: dict[int, signal.Handlers] = {} + handler: Any, +) -> dict[int, Any]: + previous: dict[int, Any] = {} for signal_number in PARENT_SIGNALS: previous[signal_number] = signal.signal(signal_number, handler) return previous def _restore_parent_signal_handlers( - previous: dict[int, signal.Handlers], + previous: dict[int, Any], ) -> None: for signal_number, handler in previous.items(): signal.signal(signal_number, handler) @@ -591,7 +1273,7 @@ def _monitor_child( state.peak_used_bytes = max( state.peak_used_bytes, state.snapshot.used_bytes ) - audit.emit( + sample_record = audit.emit( "sample", **_state_fields( state.snapshot, @@ -602,6 +1284,7 @@ def _monitor_child( "none", ), ) + audit.heartbeat(sample_record) if state.snapshot.active_swaps: return _kill_and_finish( @@ -679,7 +1362,7 @@ def run_watchdog( monotonic: Callable[[], float] | None = None, sleeper: Callable[[float], None] | None = None, ) -> int: - config.validate() + artifact_paths = config.validate() reader = reader or ProcfsReader(config.procfs_root) audit = audit or AuditLogger(sys.stderr) launcher = launcher or subprocess.Popen @@ -688,6 +1371,20 @@ def run_watchdog( monotonic = monotonic or time.monotonic sleeper = sleeper or time.sleep + if artifact_paths is not None: + try: + audit.open_persistent(artifact_paths.audit) + except ArtifactError as exc: + return _emit_final( + audit, + "lease_error", + EXIT_LEASE_ERROR, + "cannot initialize watchdog artifacts", + None, + None, + error=f"{exc.component}: {exc}", + ) + try: snapshot = reader.read_snapshot() except ProcfsError as exc: @@ -701,20 +1398,32 @@ def run_watchdog( error=str(exc), ) - audit.emit( - "preflight", - **_state_fields( + try: + audit.emit( + "preflight", + **_state_fields( + snapshot, + snapshot.used_bytes, + None, + None, + "not_created", + "none", + ), + soft_bytes=config.soft_bytes, + emergency_bytes=config.emergency_bytes, + strict_ceiling_bytes=STRICT_CEILING_BYTES, + ) + except ArtifactError as exc: + audit.disable_component(exc.component) + return _emit_final( + audit, + "lease_error", + EXIT_LEASE_ERROR, + "cannot write watchdog preflight audit", snapshot, snapshot.used_bytes, - None, - None, - "not_created", - "none", - ), - soft_bytes=config.soft_bytes, - emergency_bytes=config.emergency_bytes, - strict_ceiling_bytes=STRICT_CEILING_BYTES, - ) + error=f"{exc.component}: {exc}", + ) if snapshot.active_swaps: return _emit_final( @@ -748,21 +1457,51 @@ def run_watchdog( signal.SIG_BLOCK, PARENT_SIGNALS ) mask_restored = False - previous_handlers: dict[int, signal.Handlers] = {} + previous_handlers: dict[int, Any] = {} child: ProcessHandle | None = None state = RuntimeState(snapshot, snapshot.used_bytes) try: launch_mask = previous_mask + lease_manager = ( + LeaseManager(config, artifact_paths) + if artifact_paths is not None + else None + ) def restore_child_signal_mask() -> None: signal.pthread_sigmask(signal.SIG_SETMASK, launch_mask) try: - child = launcher( - config.command, - start_new_session=True, - preexec_fn=restore_child_signal_mask, - ) + if lease_manager is not None: + child_environment = os.environ.copy() + child_environment.update( + { + "STRIX_MEMORY_WATCHDOG_LEASE_PATH": str( + lease_manager.lease_path + ), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH": str( + lease_manager.heartbeat_path + ), + "STRIX_MEMORY_WATCHDOG_AUDIT_PATH": str( + lease_manager.audit_path + ), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS": ( + str(config.heartbeat_max_age_seconds) + ), + } + ) + child = launcher( + config.command, + start_new_session=True, + preexec_fn=restore_child_signal_mask, + env=child_environment, + ) + else: + child = launcher( + config.command, + start_new_session=True, + preexec_fn=restore_child_signal_mask, + ) except (OSError, ValueError) as exc: detail = getattr(exc, "strerror", None) or str(exc) return _emit_final( @@ -778,6 +1517,9 @@ def restore_child_signal_mask() -> None: previous_handlers = _set_parent_signal_handlers( _raise_parent_signal ) + if lease_manager is not None: + lease_manager.start(child) + audit.lease_manager = lease_manager signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) mask_restored = True audit.emit( @@ -803,8 +1545,41 @@ def restore_child_signal_mask() -> None: monotonic, sleeper, ) + except ArtifactError as exc: + _set_parent_signal_handlers(signal.SIG_IGN) + if exc.component == "audit": + audit.disable_component(exc.component) + if child is None: + return _emit_final( + audit, + "lease_error", + EXIT_LEASE_ERROR, + "watchdog artifact initialization failed", + state.snapshot, + state.peak_used_bytes, + error=f"{exc.component}: {exc}", + ) + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "lease_error", + EXIT_LEASE_ERROR, + "watchdog artifact update failed", + signal.SIGTERM, + config.grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + error=f"{exc.component}: {exc}", + ) except ParentSignal as exc: + if audit.finalized: + return audit.final_exit_code _set_parent_signal_handlers(signal.SIG_IGN) + assert child is not None signal_name = signal.Signals(exc.signal_number).name return _graceful_cleanup( audit, @@ -822,7 +1597,19 @@ def restore_child_signal_mask() -> None: sleeper, ) except Exception as exc: + if audit.finalized: + return audit.final_exit_code _set_parent_signal_handlers(signal.SIG_IGN) + if child is None: + return _emit_final( + audit, + "internal_error", + EXIT_INTERNAL_ERROR, + "unexpected pre-launch exception", + state.snapshot, + state.peak_used_bytes, + error=f"{type(exc).__name__}: {exc}", + ) return _graceful_cleanup( audit, child, @@ -900,6 +1687,39 @@ def parse_args(argv: Sequence[str]) -> WatchdogConfig: default=DEFAULT_SAMPLE_INTERVAL_SECONDS, help="procfs sampling interval (default: 1)", ) + parser.add_argument( + "--lease-path", + type=Path, + help=( + "atomically publish the watchdog-owned lease JSON; requires " + "--heartbeat-path and --audit-path" + ), + ) + parser.add_argument( + "--heartbeat-path", + type=Path, + help=( + "atomically update watchdog heartbeat JSON on every sample; " + "requires --lease-path and --audit-path" + ), + ) + parser.add_argument( + "--audit-path", + type=Path, + help=( + "create a persistent JSONL audit in addition to standard error; " + "requires --lease-path and --heartbeat-path" + ), + ) + parser.add_argument( + "--heartbeat-max-age-seconds", + type=_positive_float, + default=DEFAULT_HEARTBEAT_MAX_AGE_SECONDS, + help=( + "maximum heartbeat age accepted by a matching harness " + "(default: 5)" + ), + ) parser.add_argument( "command", nargs=argparse.REMAINDER, @@ -916,6 +1736,10 @@ def parse_args(argv: Sequence[str]) -> WatchdogConfig: emergency_bytes=args.emergency_gib * GIB, grace_seconds=args.grace_seconds, sample_interval_seconds=args.sample_interval_seconds, + lease_path=args.lease_path, + heartbeat_path=args.heartbeat_path, + audit_path=args.audit_path, + heartbeat_max_age_seconds=args.heartbeat_max_age_seconds, ) @@ -934,6 +1758,18 @@ def main(argv: Sequence[str] | None = None) -> int: None, error=str(exc), ) + except Exception as exc: + return _emit_final( + audit, + "internal_error", + EXIT_INTERNAL_ERROR, + "unexpected watchdog error", + None, + None, + error=f"{type(exc).__name__}: {exc}", + ) + finally: + audit.close() if __name__ == "__main__": diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index eca818da0f6f..81c10f84c38f 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import hashlib import io import json import os @@ -80,7 +81,7 @@ def poll(self) -> int | None: def wait(self, timeout: float | None = None) -> int: if self.returncode is None: - raise subprocess.TimeoutExpired("fake", timeout) + raise subprocess.TimeoutExpired("fake", timeout or 0.0) return self.returncode @@ -225,6 +226,33 @@ def _write_procfs_fixture(root: Path) -> None: encoding="utf-8", ) + @staticmethod + def _lease_arguments(root: Path) -> list[str]: + return [ + "--lease-path", + str(root / "lease.json"), + "--heartbeat-path", + str(root / "heartbeat.json"), + "--audit-path", + str(root / "persistent-audit.jsonl"), + ] + + @staticmethod + def _proc_stat( + process_id: int, + parent_id: int, + process_group_id: int, + start_time_ticks: int, + ) -> str: + fields = [ + "S", + str(parent_id), + str(process_group_id), + *(["0"] * 16), + str(start_time_ticks), + ] + return f"{process_id} (python) {' '.join(fields)}\n" + def test_parent_signals_leave_no_child_or_grandchild(self) -> None: child_code = ( "import os,signal,sys,time;" @@ -249,14 +277,15 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: root = Path(temp_dir) pid_file = root / "pids" self._write_procfs_fixture(root) - audit_path = root / "audit.jsonl" - with audit_path.open("w", encoding="utf-8") as audit: + stderr_path = root / "stderr.jsonl" + with stderr_path.open("w", encoding="utf-8") as audit: wrapper = subprocess.Popen( [ sys.executable, str(SCRIPT_PATH), "--procfs-root", str(root), + *self._lease_arguments(root), "--grace-seconds", "0.2", "--sample-interval-seconds", @@ -304,7 +333,7 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: ) records = [ json.loads(line) - for line in audit_path.read_text( + for line in stderr_path.read_text( encoding="utf-8" ).splitlines() ] @@ -344,6 +373,32 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: self.assertFalse( self._process_is_running(process_id) ) + lease = json.loads( + (root / "lease.json").read_text( + encoding="utf-8" + ) + ) + heartbeat = json.loads( + (root / "heartbeat.json").read_text( + encoding="utf-8" + ) + ) + persistent_records = [ + json.loads(line) + for line in ( + root / "persistent-audit.jsonl" + ).read_text(encoding="utf-8").splitlines() + ] + self.assertEqual(lease["state"], "final") + self.assertEqual( + lease["final"]["classification"], + "parent_signal", + ) + self.assertEqual(heartbeat["state"], "final") + self.assertEqual( + persistent_records[-1]["classification"], + "parent_signal", + ) def test_child_sigterm_handler_exits_without_escalation(self) -> None: child_code = ( @@ -620,27 +675,124 @@ def test_configuration_rejects_non_finite_timing(self) -> None: with self.assertRaisesRegex(ValueError, "grace period"): config.validate() + def test_final_record_survives_artifact_failures(self) -> None: + class FailingLease: + def finalize(self, record: dict[str, Any]) -> None: + raise watchdog.ArtifactError("lease", "write failed") + + class FailingAudit(io.StringIO): + def write(self, value: str) -> int: + raise OSError("write failed") + + for component in ("lease", "audit"): + with self.subTest(component=component): + stream = io.StringIO() + audit = watchdog.AuditLogger(stream) + if component == "lease": + setattr(audit, "lease_manager", FailingLease()) + else: + audit.persistent_stream = FailingAudit() + result = watchdog._emit_final( + audit, + "child_exit", + 0, + "child exited", + snapshot(50), + 50, + ) + records = [ + json.loads(line) + for line in stream.getvalue().splitlines() + ] + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual( + records[-1]["classification"], "lease_error" + ) + self.assertTrue(audit.finalized) + self.assertEqual( + audit.final_exit_code, watchdog.EXIT_LEASE_ERROR + ) + + def test_invalid_artifact_path_emits_configuration_final(self) -> None: + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--lease-path", + "~strix-watchdog-user-does-not-exist/lease.json", + "--heartbeat-path", + "/tmp/heartbeat.json", + "--audit-path", + "/tmp/audit.jsonl", + "--", + sys.executable, + "-c", + "pass", + ], + capture_output=True, + check=False, + text=True, + ) + + self.assertEqual(result.returncode, watchdog.EXIT_PROCFS_ERROR) + final = json.loads(result.stderr.splitlines()[-1]) + self.assertEqual(final["classification"], "configuration_error") + def test_cli_fixture_launches_command_and_propagates_exit(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) + child_result_path = root / "child-result.json" self._write_procfs_fixture(root) + child_code = ( + "import json,os,sys,time\n" + "keys=('STRIX_MEMORY_WATCHDOG_LEASE_PATH'," + "'STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH'," + "'STRIX_MEMORY_WATCHDOG_AUDIT_PATH')\n" + "deadline=time.monotonic()+5\n" + "while True:\n" + " try:\n" + " with open(os.environ[keys[0]],encoding='utf-8') as stream:\n" + " lease=json.load(stream)\n" + " break\n" + " except (OSError,json.JSONDecodeError):\n" + " if time.monotonic()>=deadline: raise\n" + " time.sleep(0.01)\n" + "assert os.getpgrp()==lease['child_process_group_id']\n" + "open(sys.argv[1],'w').write(json.dumps({" + "key:os.environ[key] for key in keys}))\n" + "raise SystemExit(23)\n" + ) result = subprocess.run( [ sys.executable, str(SCRIPT_PATH), "--procfs-root", str(root), + *self._lease_arguments(root), "--sample-interval-seconds", "0.01", "--", sys.executable, "-c", - "raise SystemExit(23)", + child_code, + str(child_result_path), ], capture_output=True, check=False, text=True, ) + lease = json.loads( + (root / "lease.json").read_text(encoding="utf-8") + ) + persistent_records = [ + json.loads(line) + for line in ( + root / "persistent-audit.jsonl" + ).read_text(encoding="utf-8").splitlines() + ] + child_result = json.loads( + child_result_path.read_text(encoding="utf-8") + ) self.assertEqual(result.returncode, 23) records = [ @@ -648,6 +800,277 @@ def test_cli_fixture_launches_command_and_propagates_exit(self) -> None: ] self.assertEqual(records[-1]["classification"], "child_exit") self.assertEqual(records[-1]["child_returncode"], 23) + self.assertEqual(lease["format"], watchdog.LEASE_FORMAT) + self.assertEqual(lease["version"], watchdog.LEASE_VERSION) + self.assertEqual(lease["state"], "final") + self.assertEqual(lease["soft_bytes"], 116 * 1024**3) + self.assertEqual( + lease["emergency_bytes"], 118 * 1024**3 + ) + self.assertEqual( + lease["child_command_sha256"], + watchdog._command_sha256( + ( + sys.executable, + "-c", + child_code, + str(child_result_path), + ) + ), + ) + self.assertEqual( + lease["final"]["classification"], "child_exit" + ) + self.assertEqual( + persistent_records[-1]["classification"], "child_exit" + ) + self.assertEqual( + child_result["STRIX_MEMORY_WATCHDOG_LEASE_PATH"], + str((root / "lease.json").resolve()), + ) + self.assertEqual( + child_result["STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH"], + str((root / "heartbeat.json").resolve()), + ) + self.assertEqual( + child_result["STRIX_MEMORY_WATCHDOG_AUDIT_PATH"], + str((root / "persistent-audit.jsonl").resolve()), + ) + + def test_existing_lease_fails_closed_and_stops_child(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self._write_procfs_fixture(root) + lease_path = root / "lease.json" + lease_path.write_text("untrusted\n", encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.1", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + "import time;time.sleep(30)", + ], + capture_output=True, + check=False, + text=True, + timeout=5, + ) + records = [ + json.loads(line) for line in result.stderr.splitlines() + ] + child_pid = records[-1]["child_pid"] + self.assertEqual( + lease_path.read_text(encoding="utf-8"), "untrusted\n" + ) + + self.assertEqual(result.returncode, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(records[-1]["classification"], "lease_error") + self.assertFalse(self._process_is_running(child_pid)) + + def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + process_root = root / "proc" + watchdog_pid = 1200 + child_pid = 1300 + current_pid = 1400 + watchdog_start_ticks = 456789 + script_path = root / "watchdog.py" + script_path.write_text("print('watchdog')\n", encoding="utf-8") + cmdline = ( + b"/usr/bin/python3\0watchdog.py\0--lease-path\0" + ) + for process_id, parent_id, group_id, start_ticks in ( + (watchdog_pid, 1, watchdog_pid, watchdog_start_ticks), + (child_pid, watchdog_pid, child_pid, 456790), + (current_pid, child_pid, child_pid, 456791), + ): + process_dir = process_root / str(process_id) + process_dir.mkdir(parents=True) + (process_dir / "stat").write_text( + self._proc_stat( + process_id, + parent_id, + group_id, + start_ticks, + ), + encoding="utf-8", + ) + (process_root / str(watchdog_pid) / "cwd").symlink_to( + root, target_is_directory=True + ) + (process_root / str(watchdog_pid) / "cmdline").write_bytes( + cmdline + ) + + lease_path = root / "lease.json" + heartbeat_path = root / "heartbeat.json" + audit_path = root / "audit.jsonl" + audit_path.write_text( + '{"event":"child_started","timestamp":"2026-01-01T00:00:00Z"}\n', + encoding="utf-8", + ) + command = ["python3", "run_matrix.py"] + lease = { + "format": watchdog.LEASE_FORMAT, + "version": watchdog.LEASE_VERSION, + "lease_id": "test-lease", + "state": "active", + "watchdog_pid": watchdog_pid, + "watchdog_start_time_utc": "2026-01-01T00:00:00.000Z", + "watchdog_start_time_ticks": watchdog_start_ticks, + "watchdog_command_sha256": hashlib.sha256( + cmdline + ).hexdigest(), + "watchdog_script_path": str(script_path), + "watchdog_script_sha256": hashlib.sha256( + script_path.read_bytes() + ).hexdigest(), + "soft_bytes": watchdog.DEFAULT_SOFT_BYTES, + "emergency_bytes": watchdog.DEFAULT_EMERGENCY_BYTES, + "strict_ceiling_bytes": watchdog.STRICT_CEILING_BYTES, + "child_pid": child_pid, + "child_process_group_id": child_pid, + "command": command, + "child_command_sha256": watchdog._command_sha256( + command + ), + "heartbeat_path": str(heartbeat_path), + "max_heartbeat_age_seconds": 5.0, + "audit_path": str(audit_path), + "procfs_root": "/proc", + } + heartbeat = { + "format": watchdog.HEARTBEAT_FORMAT, + "version": watchdog.HEARTBEAT_VERSION, + "lease_id": "test-lease", + "sequence": 4, + "state": "active", + "updated_at": "2026-01-01T00:00:01.000Z", + "updated_monotonic_ns": 9_000_000_000, + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": ( + watchdog_start_ticks + ), + "child_pid": child_pid, + "child_process_group_id": child_pid, + } + lease_path.write_text(json.dumps(lease), encoding="utf-8") + heartbeat_path.write_text( + json.dumps(heartbeat), encoding="utf-8" + ) + + validated = watchdog.validate_active_lease( + lease_path, + expected_script_path=script_path, + expected_command_sha256=watchdog._command_sha256( + command + ), + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=5.0, + current_process_id=current_pid, + process_procfs_root=process_root, + monotonic_ns=lambda: 10_000_000_000, + ) + self.assertEqual(validated["lease_id"], "test-lease") + + with self.subTest("tampered script SHA"): + tampered = dict(lease) + tampered["watchdog_script_sha256"] = "0" * 64 + lease_path.write_text( + json.dumps(tampered), encoding="utf-8" + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, "script SHA" + ): + watchdog.validate_active_lease( + lease_path, + expected_script_path=script_path, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + current_process_id=current_pid, + process_procfs_root=process_root, + monotonic_ns=lambda: 10_000_000_000, + ) + + with self.subTest("stale heartbeat"): + lease_path.write_text( + json.dumps(lease), encoding="utf-8" + ) + heartbeat["updated_monotonic_ns"] = 1 + heartbeat_path.write_text( + json.dumps(heartbeat), encoding="utf-8" + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, "heartbeat is stale" + ): + watchdog.validate_active_lease( + lease_path, + expected_script_path=script_path, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + current_process_id=current_pid, + process_procfs_root=process_root, + monotonic_ns=lambda: 10_000_000_000, + ) + + with self.subTest("arbitrary heartbeat"): + heartbeat["updated_monotonic_ns"] = 9_000_000_000 + heartbeat["lease_id"] = "helper-lease" + heartbeat_path.write_text( + json.dumps(heartbeat), encoding="utf-8" + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "heartbeat identity", + ): + watchdog.validate_active_lease( + lease_path, + expected_script_path=script_path, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + current_process_id=current_pid, + process_procfs_root=process_root, + monotonic_ns=lambda: 10_000_000_000, + ) + + with self.subTest("outside process group"): + heartbeat["lease_id"] = "test-lease" + heartbeat_path.write_text( + json.dumps(heartbeat), encoding="utf-8" + ) + (process_root / str(current_pid) / "stat").write_text( + self._proc_stat( + current_pid, + child_pid, + 9999, + 456791, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "outside the monitored process group", + ): + watchdog.validate_active_lease( + lease_path, + expected_script_path=script_path, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + current_process_id=current_pid, + process_procfs_root=process_root, + monotonic_ns=lambda: 10_000_000_000, + ) def test_zero_swap_gate_launches_and_propagates_child_exit(self) -> None: harness = Harness([snapshot(50)], FakeProcess(returncode=37)) @@ -702,13 +1125,29 @@ def exit_on_term(process: FakeProcess, signal_number: int) -> None: signal_handler=exit_on_term, ) - result = harness.run() + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + result = harness.run( + lease_path=root / "lease.json", + heartbeat_path=root / "heartbeat.json", + audit_path=root / "audit.jsonl", + ) + harness.audit.close() + heartbeat = json.loads( + (root / "heartbeat.json").read_text(encoding="utf-8") + ) + lease = json.loads( + (root / "lease.json").read_text(encoding="utf-8") + ) self.assertEqual(result, watchdog.EXIT_SOFT_LIMIT) self.assertEqual(harness.signals, [signal.SIGTERM]) self.assertEqual( harness.records()[-1]["classification"], "soft_limit" ) + self.assertEqual(heartbeat["state"], "final") + self.assertEqual(heartbeat["sequence"], 3) + self.assertEqual(lease["final"]["classification"], "soft_limit") def test_emergency_limit_sends_sigkill(self) -> None: def exit_on_kill(process: FakeProcess, signal_number: int) -> None: @@ -804,7 +1243,23 @@ def exit_on_kill(process: FakeProcess, signal_number: int) -> None: signal_handler=exit_on_kill, ) - result = harness.run() + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + result = harness.run( + lease_path=root / "lease.json", + heartbeat_path=root / "heartbeat.json", + audit_path=root / "audit.jsonl", + ) + harness.audit.close() + lease = json.loads( + (root / "lease.json").read_text(encoding="utf-8") + ) + persistent_records = [ + json.loads(line) + for line in (root / "audit.jsonl").read_text( + encoding="utf-8" + ).splitlines() + ] self.assertEqual(result, watchdog.EXIT_INTERNAL_ERROR) self.assertEqual( @@ -814,6 +1269,10 @@ def exit_on_kill(process: FakeProcess, signal_number: int) -> None: final = harness.records()[-1] self.assertEqual(final["classification"], "internal_error") self.assertIn("RuntimeError: unexpected", final["error"]) + self.assertEqual(lease["final"]["classification"], "internal_error") + self.assertEqual( + persistent_records[-1]["classification"], "internal_error" + ) def test_launch_failure_is_explicit(self) -> None: harness = Harness([snapshot(50)], FakeProcess()) @@ -823,7 +1282,7 @@ def fail_launch( ) -> FakeProcess: raise FileNotFoundError(2, "No such file or directory") - harness.launcher = fail_launch + setattr(harness, "launcher", fail_launch) result = harness.run() From c4598ee747fdb811b474c0b41bdb38eeda3bbc0c Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 23:04:21 -0700 Subject: [PATCH 14/56] scripts : harden watchdog fail-closed lease Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 28 +- scripts/strix_memory_watchdog.py | 862 ++++++++++++++++++++++++---- tests/test_strix_memory_watchdog.py | 682 ++++++++++++++++++---- 3 files changed, 1339 insertions(+), 233 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 5565716a0a3e..49a1c05547fc 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -38,34 +38,40 @@ Use all three artifact options together when another process must prove that it python3 tools/deepseek-v41-trace/run_matrix.py ``` -The watchdog creates the persistent audit before launch, then atomically creates the lease and heartbeat after `Popen` returns. Existing artifact paths are rejected rather than overwritten. The child receives the resolved paths through `STRIX_MEMORY_WATCHDOG_LEASE_PATH`, `STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH`, and `STRIX_MEMORY_WATCHDOG_AUDIT_PATH`. It also receives `STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS`. +The watchdog creates and exclusively locks the persistent audit before launch. It then starts an internal guardian as the new session and process-group leader; the guardian starts the supplied command in that same group without inheriting the private control pipe. After the guardian reports the payload PID, the watchdog atomically creates the lease and heartbeat. Existing artifact paths are rejected rather than overwritten. The payload receives the resolved paths through `STRIX_MEMORY_WATCHDOG_LEASE_PATH`, `STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH`, and `STRIX_MEMORY_WATCHDOG_AUDIT_PATH`. It also receives `STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS`. The child can run before the first atomic lease rename. A matching preflight must retry the inherited lease path for a bounded interval and fail closed if a complete valid lease does not appear. It must not accept a lease path supplied separately by the operator. Lease format `strix-memory-watchdog-lease`, version 1, contains: - `lease_id` and active/final `state` -- `watchdog_pid`, `watchdog_start_time_utc`, Linux `watchdog_start_time_ticks`, `watchdog_command_sha256`, `watchdog_script_path`, and `watchdog_script_sha256` +- `watchdog_pid`, `watchdog_start_time_utc`, Linux `watchdog_start_time_ticks`, `watchdog_executable_path`, `watchdog_command_sha256`, `watchdog_script_path`, and `watchdog_script_sha256` - exact `soft_bytes`, `emergency_bytes`, and `strict_ceiling_bytes` - `procfs_root` -- `child_pid`, `child_process_group_id`, `command`, and `child_command_sha256` +- `guardian_pid`, payload `child_pid`, `child_process_group_id`, `command`, and `child_command_sha256` - `heartbeat_path`, `max_heartbeat_age_seconds`, and `audit_path` +- device, inode, owner, and mode identity for atomic JSON artifacts, plus the watchdog-held audit descriptor identity - the authoritative `final` audit record after termination -Heartbeat format `strix-memory-watchdog-heartbeat`, version 1, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample atomically replaces the heartbeat and includes the complete sample audit record. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. +Heartbeat format `strix-memory-watchdog-heartbeat`, version 1, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample first checks swap and memory thresholds, pulses the guardian through the private nonblocking pipe, then atomically replaces the heartbeat with the complete sample audit record and its persistent-audit record hash. A blocked audit or heartbeat write cannot delay the emergency signal. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. + +The guardian uses Linux `PR_SET_PDEATHSIG` with a parent-race check. It kills its process group on watchdog death, control-pipe EOF/error, or a missed pulse deadline, including a stopped or wedged watchdog. The payload must call `start_process_group_lease_guard()` before it starts exporter descendants. This validates the lease with bounded startup retries, arms a second parent-death link to the guardian, and starts a thread that kills the process group if the watchdog evidence becomes stale or invalid. A matching Linux preflight must verify all of the following: - The inherited lease, heartbeat, and audit paths match the paths inside the lease. -- The expected repository script path hashes to `watchdog_script_sha256`. -- `/proc//stat` start ticks and `/proc//cmdline` SHA-256 match the lease. -- The watchdog command line names the expected script. -- The current process group equals `child_process_group_id`, whose leader is `child_pid` and whose parent is `watchdog_pid`. +- `/proc//exe` is the exact expected Python executable and argv position 1 is the exact repository watchdog script. `-c`, `-m`, helper-script, inert-argument, and interpreter-option substitutions are rejected. +- The watchdog command line itself supplies the exact 116/118 GiB thresholds, `/proc`, inherited artifact paths, and command after `--`; the lease cannot override those expectations. +- `/proc//stat` start ticks and `/proc//cmdline` SHA-256 match the lease and remain stable across validation. A pidfd is held during validation when Linux provides `pidfd_open`. +- The topology is watchdog parent -> guardian process-group leader -> payload child. The current process must be inside `child_process_group_id`. - The command identity is expected, the procfs root is `/proc`, and thresholds are exactly 116 GiB soft, 118 GiB emergency, and 120 GiB strict ceiling for the final run. -- The heartbeat identity matches the lease and its monotonic timestamp is not older than `max_heartbeat_age_seconds`. -- The persistent audit exists and contains watchdog JSONL records. +- Lease and heartbeat files are regular, mode 0600, owned by the current UID, opened with `O_NOFOLLOW`, and match their recorded device/inode identity. +- The heartbeat identity matches the lease, its monotonic timestamp is not older than `max_heartbeat_age_seconds`, and its audit-record hash exists in the persistent audit. +- The persistent audit matches the watchdog-held descriptor device/inode and remains exclusively locked by the live watchdog. + +These checks reject accidental or helper-process substitution and make regular-file heartbeat forgery unable to keep the process group alive after private pulses stop. They are not a security boundary against intentionally hostile code running as the same UID; use a separately owned systemd user service or cgroup if that threat is in scope. -These checks bind the validation process to the live canonical watchdog. A standalone heartbeat helper has a different PID, start time, command line, script hash, and process group and cannot satisfy the lease. +The guardian controls only the process group. A payload that deliberately calls `setsid()` can escape it. The correctness harness must not do that. If arbitrary payload code is in scope, launch the watchdog in a service/cgroup configured to kill every member when the unit stops. Exit classifications are authoritative in the final JSON record. Operational failures use these exit codes: diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index 0bb11a0622c3..dd185a00f5f2 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -3,15 +3,20 @@ from __future__ import annotations import argparse +import ctypes +import fcntl import hashlib import json import math import os import re import secrets +import select import signal +import stat import subprocess import sys +import threading import time from collections.abc import Callable, Sequence from dataclasses import dataclass @@ -32,6 +37,8 @@ LEASE_VERSION = 1 HEARTBEAT_FORMAT = "strix-memory-watchdog-heartbeat" HEARTBEAT_VERSION = 1 +PR_SET_PDEATHSIG = 1 +LEASE_GUARD_SIGNAL = signal.SIGUSR1 EXIT_PROCFS_ERROR = 2 EXIT_SWAP_ACTIVE = 3 @@ -82,6 +89,42 @@ def wait(self, timeout: float | None = None) -> int: ... +@dataclass +class GuardianProcess: + process: subprocess.Popen[bytes] + payload_pid: int + pulse_fd: int + + @property + def pid(self) -> int: + return self.process.pid + + def poll(self) -> int | None: + return self.process.poll() + + def wait(self, timeout: float | None = None) -> int: + return self.process.wait(timeout=timeout) + + def pulse(self) -> None: + try: + os.write(self.pulse_fd, b"\0") + except BlockingIOError as exc: + raise ProcessGroupError( + "guardian pulse pipe is blocked" + ) from exc + except OSError as exc: + detail = exc.strerror or str(exc) + raise ProcessGroupError( + f"cannot pulse guardian: {detail}" + ) from exc + + def close(self) -> None: + try: + os.close(self.pulse_fd) + except OSError: + pass + + @dataclass(frozen=True) class HostSnapshot: total_bytes: int @@ -139,6 +182,14 @@ def validate(self) -> ArtifactPaths | None: or self.sample_interval_seconds <= 0 ): raise ValueError("sample interval must be greater than zero") + if ( + not math.isfinite(self.heartbeat_max_age_seconds) + or self.heartbeat_max_age_seconds + <= self.sample_interval_seconds + ): + raise ValueError( + "heartbeat max age must be greater than sample interval" + ) lease_paths = ( self.lease_path, self.heartbeat_path, @@ -168,14 +219,6 @@ def validate(self) -> ArtifactPaths | None: raise ValueError( "lease, heartbeat, and audit paths must be distinct" ) - if ( - not math.isfinite(self.heartbeat_max_age_seconds) - or self.heartbeat_max_age_seconds - <= self.sample_interval_seconds - ): - raise ValueError( - "heartbeat max age must be greater than sample interval" - ) return paths return None @@ -278,6 +321,185 @@ def _command_sha256(command: Sequence[str]) -> str: return _sha256_bytes(encoded) +def _set_parent_death_signal( + signal_number: int, expected_parent_pid: int +) -> None: + if not sys.platform.startswith("linux"): + return + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(PR_SET_PDEATHSIG, signal_number, 0, 0, 0) != 0: + error_number = ctypes.get_errno() + raise OSError(error_number, os.strerror(error_number)) + if os.getppid() != expected_parent_pid: + os.kill(os.getpid(), signal.SIGKILL) + + +def _kill_own_process_group( + _signal_number: int | None = None, + _frame: object | None = None, +) -> None: + try: + os.killpg(os.getpgrp(), signal.SIGKILL) + except OSError: + os._exit(EXIT_SIGNAL_ERROR) + + +def _guardian_main( + control_fd: int, + status_fd: int, + pulse_timeout_seconds: float, + command: tuple[str, ...], +) -> int: + if not sys.platform.startswith("linux"): + return EXIT_LAUNCH_ERROR + os.set_inheritable(control_fd, False) + os.set_inheritable(status_fd, False) + signal.signal(LEASE_GUARD_SIGNAL, _kill_own_process_group) + for signal_number in PARENT_SIGNALS: + signal.signal(signal_number, signal.SIG_IGN) + _set_parent_death_signal(LEASE_GUARD_SIGNAL, os.getppid()) + + def prepare_payload() -> None: + for signal_number in PARENT_SIGNALS: + signal.signal(signal_number, signal.SIG_DFL) + + try: + payload = subprocess.Popen(command, preexec_fn=prepare_payload) + except (OSError, ValueError) as exc: + os.write( + status_fd, + json.dumps( + {"error": getattr(exc, "strerror", None) or str(exc)} + ).encode("utf-8") + + b"\n", + ) + os.close(status_fd) + return EXIT_LAUNCH_ERROR + + os.write( + status_fd, + json.dumps({"payload_pid": payload.pid}).encode("utf-8") + b"\n", + ) + os.close(status_fd) + poller = select.poll() + poller.register( + control_fd, + select.POLLIN | select.POLLHUP | select.POLLERR, + ) + deadline = time.monotonic() + pulse_timeout_seconds + while True: + remaining = max(0.0, deadline - time.monotonic()) + events = poller.poll(max(1, min(50, int(remaining * 1000)))) + for _, event_mask in events: + if event_mask & (select.POLLHUP | select.POLLERR): + _kill_own_process_group() + try: + pulse = os.read(control_fd, 65536) + except BlockingIOError: + pulse = b"" + if not pulse: + _kill_own_process_group() + deadline = time.monotonic() + pulse_timeout_seconds + if time.monotonic() >= deadline: + _kill_own_process_group() + returncode = payload.poll() + if returncode is not None: + return ( + 128 - returncode + if returncode < 0 + else returncode + ) + + +def _read_guardian_status( + descriptor: int, timeout_seconds: float +) -> int: + poller = select.poll() + poller.register(descriptor, select.POLLIN | select.POLLHUP) + deadline = time.monotonic() + timeout_seconds + content = b"" + while time.monotonic() < deadline: + events = poller.poll( + max(1, int((deadline - time.monotonic()) * 1000)) + ) + if not events: + continue + chunk = os.read(descriptor, 4096) + if not chunk: + break + content += chunk + if b"\n" in content: + break + if not content: + raise OSError("guardian did not report payload startup") + try: + status = json.loads(content.splitlines()[0]) + except (UnicodeError, json.JSONDecodeError) as exc: + raise OSError("guardian returned malformed startup status") from exc + if not isinstance(status, dict): + raise OSError("guardian returned malformed startup status") + if "error" in status: + raise OSError(str(status["error"])) + payload_pid = status.get("payload_pid") + if not isinstance(payload_pid, int): + raise OSError("guardian did not report a payload PID") + return payload_pid + + +def _launch_guardian( + command: tuple[str, ...], + environment: dict[str, str], + pulse_timeout_seconds: float, + launch_mask: set[signal.Signals], +) -> GuardianProcess: + control_read, control_write = os.pipe() + os.set_blocking(control_read, False) + os.set_blocking(control_write, False) + status_read, status_write = os.pipe() + parent_pid = os.getpid() + + def prepare_guardian() -> None: + signal.pthread_sigmask(signal.SIG_SETMASK, launch_mask) + _set_parent_death_signal(signal.SIGKILL, parent_pid) + + guardian_command = ( + sys.executable, + str(Path(__file__).resolve()), + "--internal-guardian", + str(control_read), + str(status_write), + str(pulse_timeout_seconds), + "--", + *command, + ) + try: + process = subprocess.Popen( + guardian_command, + start_new_session=True, + pass_fds=(control_read, status_write), + preexec_fn=prepare_guardian, + env=environment, + ) + finally: + os.close(control_read) + os.close(status_write) + try: + payload_pid = _read_guardian_status(status_read, 5.0) + except OSError: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5.0) + os.close(control_write) + raise + finally: + os.close(status_read) + guardian = GuardianProcess(process, payload_pid, control_write) + guardian.pulse() + return guardian + + def _read_proc_bytes(root: Path, process_id: int, name: str) -> bytes: path = root / str(process_id) / name try: @@ -321,15 +543,6 @@ def _write_json_atomic( temp_path = parent / ( f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp" ) - payload = ( - json.dumps( - value, - ensure_ascii=True, - sort_keys=True, - separators=(",", ":"), - ) - + "\n" - ).encode("utf-8") try: descriptor = os.open( temp_path, @@ -337,6 +550,23 @@ def _write_json_atomic( 0o600, ) with os.fdopen(descriptor, "wb") as stream: + file_status = os.fstat(stream.fileno()) + record = { + **value, + "file_device": file_status.st_dev, + "file_inode": file_status.st_ino, + "file_uid": file_status.st_uid, + "file_mode": stat.S_IMODE(file_status.st_mode), + } + payload = ( + json.dumps( + record, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode("utf-8") stream.write(payload) stream.flush() os.fsync(stream.fileno()) @@ -394,6 +624,11 @@ def _watchdog_identity(self) -> dict[str, object]: _, _, proc_start_time_ticks = _read_proc_stat( self.process_procfs_root, os.getpid() ) + executable_path = ( + self.process_procfs_root + / str(os.getpid()) + / "exe" + ).resolve() except (OSError, LeaseValidationError): if sys.platform.startswith("linux"): raise ArtifactError( @@ -403,11 +638,13 @@ def _watchdog_identity(self) -> dict[str, object]: cmdline = b"\0".join( os.fsencode(argument) for argument in sys.argv ) + executable_path = Path(sys.executable).resolve() return { "pid": os.getpid(), "start_time_utc": _timestamp_utc(self.wall_clock), "proc_start_time_ticks": proc_start_time_ticks, "cmdline_sha256": _sha256_bytes(cmdline), + "executable_path": str(executable_path), "script_path": str(script_path), "script_sha256": _sha256_file(script_path), } @@ -440,8 +677,16 @@ def _heartbeat_record( record["sample"] = sample return record - def start(self, child: ProcessHandle) -> None: + def start( + self, child: ProcessHandle, audit: AuditLogger + ) -> None: watchdog_identity = self._watchdog_identity() + audit_identity = audit.persistent_identity() + payload_pid = ( + child.payload_pid + if isinstance(child, GuardianProcess) + else child.pid + ) self.lease = { "format": LEASE_FORMAT, "version": LEASE_VERSION, @@ -457,6 +702,9 @@ def start(self, child: ProcessHandle) -> None: "watchdog_command_sha256": ( watchdog_identity["cmdline_sha256"] ), + "watchdog_executable_path": ( + watchdog_identity["executable_path"] + ), "watchdog_script_path": watchdog_identity["script_path"], "watchdog_script_sha256": ( watchdog_identity["script_sha256"] @@ -464,7 +712,8 @@ def start(self, child: ProcessHandle) -> None: "soft_bytes": self.config.soft_bytes, "emergency_bytes": self.config.emergency_bytes, "strict_ceiling_bytes": STRICT_CEILING_BYTES, - "child_pid": child.pid, + "guardian_pid": child.pid, + "child_pid": payload_pid, "child_process_group_id": child.pid, "command": list(self.config.command), "child_command_sha256": _command_sha256( @@ -475,11 +724,19 @@ def start(self, child: ProcessHandle) -> None: self.config.heartbeat_max_age_seconds ), "audit_path": str(self.audit_path), + "audit_device": audit_identity["device"], + "audit_inode": audit_identity["inode"], + "audit_uid": audit_identity["uid"], + "audit_mode": audit_identity["mode"], + "audit_fd": audit_identity["fd"], "procfs_root": str( self.config.procfs_root.expanduser().resolve() ), } - heartbeat = self._heartbeat_record("active") + heartbeat = self._heartbeat_record( + "active", + {"audit_record_sha256": audit.last_record_sha256}, + ) _write_json_atomic(self.heartbeat_path, heartbeat, create=True) _write_json_atomic(self.lease_path, self.lease, create=True) @@ -499,13 +756,34 @@ def finalize(self, final_record: dict[str, object]) -> None: def _read_json_object(path: Path) -> dict[str, object]: try: - value = json.loads(path.read_text(encoding="utf-8")) + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + with os.fdopen(descriptor, "r", encoding="utf-8") as stream: + file_status = os.fstat(stream.fileno()) + if ( + not stat.S_ISREG(file_status.st_mode) + or file_status.st_uid != os.getuid() + or stat.S_IMODE(file_status.st_mode) != 0o600 + ): + raise LeaseValidationError( + f"{path} has unsafe type, owner, or mode" + ) + value = json.load(stream) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise LeaseValidationError( f"cannot read valid JSON from {path}: {exc}" ) from exc if not isinstance(value, dict): raise LeaseValidationError(f"{path} must contain a JSON object") + if ( + value.get("file_device") != file_status.st_dev + or value.get("file_inode") != file_status.st_ino + or value.get("file_uid") != file_status.st_uid + or value.get("file_mode") != stat.S_IMODE(file_status.st_mode) + ): + raise LeaseValidationError(f"{path} identity does not match") return value @@ -525,16 +803,20 @@ def validate_active_lease( lease_path: Path, *, expected_script_path: Path, + expected_executable_path: Path | None = None, expected_soft_bytes: int = DEFAULT_SOFT_BYTES, expected_emergency_bytes: int = DEFAULT_EMERGENCY_BYTES, expected_procfs_root: Path = Path("/proc"), - expected_command_sha256: str | None = None, + expected_command: Sequence[str] | None = None, expected_heartbeat_path: Path | None = None, expected_audit_path: Path | None = None, expected_max_heartbeat_age_seconds: float | None = None, current_process_id: int | None = None, process_procfs_root: Path = Path("/proc"), monotonic_ns: Callable[[], int] | None = None, + pidfd_open: Callable[[int], int] | None = getattr( + os, "pidfd_open", None + ), ) -> dict[str, object]: lease_path = lease_path.expanduser().resolve() lease = _read_json_object(lease_path) @@ -587,6 +869,14 @@ def validate_active_lease( watchdog_pid = _require_int( lease.get("watchdog_pid"), "watchdog_pid" ) + pidfd: int | None = None + if pidfd_open is not None: + try: + pidfd = pidfd_open(watchdog_pid) + except OSError as exc: + raise LeaseValidationError( + "cannot open watchdog pidfd" + ) from exc watchdog_start_ticks = _require_int( lease.get("watchdog_start_time_ticks"), "watchdog_start_time_ticks", @@ -596,6 +886,28 @@ def validate_active_lease( ) if live_watchdog_start_ticks != watchdog_start_ticks: raise LeaseValidationError("watchdog process start time does not match") + expected_executable = ( + expected_executable_path or Path(sys.executable) + ).expanduser().resolve() + try: + live_executable = ( + process_procfs_root / str(watchdog_pid) / "exe" + ).resolve() + except OSError as exc: + raise LeaseValidationError( + "cannot resolve watchdog executable" + ) from exc + if ( + live_executable != expected_executable + or Path( + _require_string( + lease.get("watchdog_executable_path"), + "watchdog_executable_path", + ) + ).resolve() + != expected_executable + ): + raise LeaseValidationError("watchdog executable does not match") live_cmdline = _read_proc_bytes( process_procfs_root, watchdog_pid, "cmdline" ) @@ -604,48 +916,91 @@ def validate_active_lease( "watchdog_command_sha256", ): raise LeaseValidationError("watchdog command line does not match") - script_named = False - watchdog_cwd: Path | None = None - for raw_argument in live_cmdline.split(b"\0"): - if not raw_argument: - continue - argument_path = Path(os.fsdecode(raw_argument)).expanduser() - if not argument_path.is_absolute(): - if watchdog_cwd is None: - try: - watchdog_cwd = ( - process_procfs_root - / str(watchdog_pid) - / "cwd" - ).resolve() - except OSError as exc: - raise LeaseValidationError( - "cannot resolve watchdog working directory" - ) from exc - argument_path = watchdog_cwd / argument_path - if argument_path.resolve() == expected_script_path: - script_named = True - break - if not script_named: + argv = [ + os.fsdecode(argument) + for argument in live_cmdline.split(b"\0") + if argument + ] + if len(argv) < 2 or argv[1] in ("-c", "-m"): raise LeaseValidationError( - "watchdog command line does not name the expected script" + "watchdog script is not in executable argv position" ) + try: + watchdog_cwd = ( + process_procfs_root / str(watchdog_pid) / "cwd" + ).resolve() + except OSError as exc: + raise LeaseValidationError( + "cannot resolve watchdog working directory" + ) from exc + argv_script = Path(argv[1]).expanduser() + if not argv_script.is_absolute(): + argv_script = watchdog_cwd / argv_script + if argv_script.resolve() != expected_script_path: + raise LeaseValidationError( + "watchdog script is not in executable argv position" + ) + try: + live_config = parse_args(argv[2:]) + live_paths = live_config.validate() + except (SystemExit, ValueError) as exc: + raise LeaseValidationError( + "watchdog command line is invalid" + ) from exc + if ( + live_config.soft_bytes != expected_soft_bytes + or live_config.emergency_bytes != expected_emergency_bytes + or live_config.procfs_root.expanduser().resolve() + != expected_procfs_root.expanduser().resolve() + ): + raise LeaseValidationError( + "watchdog command-line policy does not match" + ) + if ( + live_paths is None + or live_paths.lease != lease_path + or ( + expected_heartbeat_path is not None + and live_paths.heartbeat + != expected_heartbeat_path.expanduser().resolve() + ) + or ( + expected_audit_path is not None + and live_paths.audit + != expected_audit_path.expanduser().resolve() + ) + ): + raise LeaseValidationError( + "watchdog command-line artifact paths do not match" + ) + if expected_command is not None and tuple( + expected_command + ) != live_config.command: + raise LeaseValidationError("monitored command does not match") + guardian_pid = _require_int( + lease.get("guardian_pid"), "guardian_pid" + ) child_pid = _require_int(lease.get("child_pid"), "child_pid") process_group_id = _require_int( lease.get("child_process_group_id"), "child_process_group_id", ) + guardian_parent_pid, guardian_group_id, _ = _read_proc_stat( + process_procfs_root, guardian_pid + ) child_parent_pid, child_group_id, _ = _read_proc_stat( process_procfs_root, child_pid ) if ( - child_parent_pid != watchdog_pid + guardian_parent_pid != watchdog_pid + or guardian_group_id != process_group_id + or guardian_pid != process_group_id + or child_parent_pid != guardian_pid or child_group_id != process_group_id - or process_group_id != child_pid ): raise LeaseValidationError( - "monitored child parent or process group does not match" + "watchdog, guardian, child, or process group does not match" ) command = lease.get("command") if ( @@ -660,9 +1015,8 @@ def validate_active_lease( ) if command_sha256 != _command_sha256(command): raise LeaseValidationError("monitored command SHA is invalid") - if ( - expected_command_sha256 is not None - and command_sha256 != expected_command_sha256 + if expected_command is not None and command_sha256 != _command_sha256( + expected_command ): raise LeaseValidationError("monitored command SHA does not match") @@ -725,6 +1079,13 @@ def validate_active_lease( ) _require_int(heartbeat.get("sequence"), "heartbeat.sequence") _require_string(heartbeat.get("updated_at"), "heartbeat.updated_at") + heartbeat_sample = heartbeat.get("sample") + if not isinstance(heartbeat_sample, dict): + raise LeaseValidationError("heartbeat sample is invalid") + audit_record_sha256 = _require_string( + heartbeat_sample.get("audit_record_sha256"), + "heartbeat.sample.audit_record_sha256", + ) now_monotonic_ns = (monotonic_ns or time.monotonic_ns)() age_ns = now_monotonic_ns - updated_monotonic_ns if age_ns < 0 or age_ns > int(heartbeat_max_age * 1_000_000_000): @@ -738,14 +1099,64 @@ def validate_active_lease( and audit_path != expected_audit_path.expanduser().resolve() ): raise LeaseValidationError("persistent audit path does not match") + audit_fd = _require_int(lease.get("audit_fd"), "audit_fd") + audit_device = _require_int( + lease.get("audit_device"), "audit_device" + ) + audit_inode = _require_int( + lease.get("audit_inode"), "audit_inode" + ) + audit_uid = _require_int(lease.get("audit_uid"), "audit_uid") + audit_mode = _require_int(lease.get("audit_mode"), "audit_mode") try: - first_line = next( + audit_status = audit_path.stat(follow_symlinks=False) + live_audit_status = ( + process_procfs_root + / str(watchdog_pid) + / "fd" + / str(audit_fd) + ).stat() + if ( + not stat.S_ISREG(audit_status.st_mode) + or audit_status.st_dev != audit_device + or audit_status.st_ino != audit_inode + or live_audit_status.st_dev != audit_device + or live_audit_status.st_ino != audit_inode + or audit_status.st_uid != audit_uid + or audit_uid != os.getuid() + or stat.S_IMODE(audit_status.st_mode) != audit_mode + or audit_mode != 0o600 + ): + raise LeaseValidationError( + "persistent audit identity does not match" + ) + audit_descriptor = os.open( + audit_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + try: + fcntl.flock( + audit_descriptor, + fcntl.LOCK_EX | fcntl.LOCK_NB, + ) + except BlockingIOError: + pass + else: + fcntl.flock(audit_descriptor, fcntl.LOCK_UN) + raise LeaseValidationError( + "watchdog does not hold the persistent audit lock" + ) + finally: + os.close(audit_descriptor) + audit_lines = [ line for line in audit_path.read_text( encoding="utf-8" ).splitlines() if line - ) + ] + first_line = next(iter(audit_lines)) first_record = json.loads(first_line) if ( not isinstance(first_record, dict) @@ -755,6 +1166,14 @@ def validate_active_lease( raise LeaseValidationError( "persistent audit does not contain watchdog records" ) + if not any( + _sha256_bytes((line + "\n").encode("utf-8")) + == audit_record_sha256 + for line in audit_lines + ): + raise LeaseValidationError( + "heartbeat audit record does not match persistent audit" + ) except StopIteration as exc: raise LeaseValidationError("persistent audit is empty") from exc except (UnicodeError, json.JSONDecodeError) as exc: @@ -767,9 +1186,104 @@ def validate_active_lease( raise LeaseValidationError( f"cannot inspect persistent audit {audit_path}: {exc}" ) from exc + _, _, final_watchdog_start_ticks = _read_proc_stat( + process_procfs_root, watchdog_pid + ) + if final_watchdog_start_ticks != watchdog_start_ticks: + raise LeaseValidationError( + "watchdog process changed during validation" + ) + if pidfd is not None: + os.close(pidfd) return lease +def start_process_group_lease_guard( + expected_script_path: Path, + *, + startup_timeout_seconds: float = 5.0, + expected_procfs_root: Path = Path("/proc"), + process_procfs_root: Path = Path("/proc"), +) -> threading.Thread: + try: + lease_path = Path( + os.environ["STRIX_MEMORY_WATCHDOG_LEASE_PATH"] + ).resolve() + heartbeat_path = Path( + os.environ["STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH"] + ).resolve() + audit_path = Path( + os.environ["STRIX_MEMORY_WATCHDOG_AUDIT_PATH"] + ).resolve() + max_age_seconds = float( + os.environ[ + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS" + ] + ) + except (KeyError, ValueError) as exc: + raise LeaseValidationError( + "watchdog artifact environment is missing or invalid" + ) from exc + current_cmdline = _read_proc_bytes( + process_procfs_root, os.getpid(), "cmdline" + ) + expected_command = tuple( + os.fsdecode(argument) + for argument in current_cmdline.split(b"\0") + if argument + ) + deadline = time.monotonic() + startup_timeout_seconds + while True: + try: + lease = validate_active_lease( + lease_path, + expected_script_path=expected_script_path, + expected_procfs_root=expected_procfs_root, + expected_command=expected_command, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=max_age_seconds, + process_procfs_root=process_procfs_root, + ) + break + except LeaseValidationError: + if time.monotonic() >= deadline: + raise + time.sleep(0.01) + + guardian_pid = _require_int( + lease.get("guardian_pid"), "guardian_pid" + ) + signal.signal(LEASE_GUARD_SIGNAL, _kill_own_process_group) + _set_parent_death_signal(LEASE_GUARD_SIGNAL, guardian_pid) + + def monitor() -> None: + interval = min(1.0, max_age_seconds / 3) + while True: + time.sleep(interval) + try: + validate_active_lease( + lease_path, + expected_script_path=expected_script_path, + expected_procfs_root=expected_procfs_root, + expected_command=expected_command, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=max_age_seconds, + process_procfs_root=process_procfs_root, + ) + except LeaseValidationError: + _kill_own_process_group() + + guard = threading.Thread( + target=monitor, + name="strix-watchdog-lease-guard", + daemon=True, + ) + guard.start() + return guard + + class AuditLogger: def __init__( self, @@ -782,15 +1296,22 @@ def __init__( self.lease_manager: LeaseManager | None = None self.finalized = False self.final_exit_code = EXIT_INTERNAL_ERROR + self.last_record_sha256: str | None = None def open_persistent(self, path: Path) -> None: resolved_path = path.expanduser().resolve() try: descriptor = os.open( resolved_path, - os.O_CREAT | os.O_EXCL | os.O_WRONLY, + os.O_CREAT + | os.O_EXCL + | os.O_WRONLY + | getattr(os, "O_NOFOLLOW", 0), 0o600, ) + fcntl.flock( + descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB + ) self.persistent_stream = os.fdopen( descriptor, "w", encoding="utf-8" ) @@ -801,6 +1322,20 @@ def open_persistent(self, path: Path) -> None: f"cannot create persistent audit {resolved_path}: {detail}", ) from exc + def persistent_identity(self) -> dict[str, int]: + if self.persistent_stream is None: + raise ArtifactError( + "audit", "persistent audit is not open" + ) + file_status = os.fstat(self.persistent_stream.fileno()) + return { + "device": file_status.st_dev, + "inode": file_status.st_ino, + "uid": file_status.st_uid, + "mode": stat.S_IMODE(file_status.st_mode), + "fd": self.persistent_stream.fileno(), + } + def close(self) -> None: if self.persistent_stream is not None: self.persistent_stream.close() @@ -824,6 +1359,7 @@ def emit(self, event: str, **fields: object) -> dict[str, object]: ) self.stream.write(line) self.stream.flush() + self.last_record_sha256 = _sha256_bytes(line.encode("utf-8")) if self.persistent_stream is not None: try: self.persistent_stream.write(line) @@ -839,7 +1375,12 @@ def emit(self, event: str, **fields: object) -> dict[str, object]: def heartbeat(self, sample: dict[str, object]) -> None: if self.lease_manager is not None: - self.lease_manager.update_heartbeat(sample) + self.lease_manager.update_heartbeat( + { + **sample, + "audit_record_sha256": self.last_record_sha256, + } + ) def finalize(self, record: dict[str, object]) -> None: if self.lease_manager is not None: @@ -952,6 +1493,30 @@ def _signal_process_group(process_group_id: int, signal_number: int) -> str: def _process_group_alive(process_group_id: int) -> bool: + if sys.platform.startswith("linux"): + try: + process_paths = Path("/proc").iterdir() + for process_path in process_paths: + if not process_path.name.isdigit(): + continue + try: + content = ( + process_path / "stat" + ).read_text(encoding="utf-8") + close_paren = content.rfind(")") + fields = content[close_paren + 1:].split() + if ( + close_paren >= 0 + and len(fields) >= 3 + and fields[0] != "Z" + and int(fields[2]) == process_group_id + ): + return True + except (OSError, UnicodeError, ValueError): + continue + return False + except OSError: + pass try: os.killpg(process_group_id, 0) except ProcessLookupError: @@ -1012,18 +1577,23 @@ def _kill_and_finish( str(exc), ) - audit.emit( - "process_group_signal", - **_state_fields( - snapshot, - peak_used_bytes, - child, - child.poll(), - group_status, - reason, - ), - signal="SIGKILL", - ) + artifact_error: ArtifactError | None = None + try: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + group_status, + reason, + ), + signal="SIGKILL", + ) + except ArtifactError as exc: + artifact_error = exc + audit.disable_component(exc.component) try: child_returncode = child.wait(timeout=5.0) except subprocess.TimeoutExpired as exc: @@ -1039,6 +1609,12 @@ def _kill_and_finish( "sigkill_timeout", str(exc), ) + if artifact_error is not None: + classification = "lease_error" + exit_code = EXIT_LEASE_ERROR + error = f"{artifact_error.component}: {artifact_error}" + else: + error = None return _emit_final( audit, classification, @@ -1049,6 +1625,7 @@ def _kill_and_finish( child, child_returncode, group_status, + error, ) @@ -1071,23 +1648,28 @@ def _graceful_cleanup( error: str | None = None, ) -> int: escalated = False + artifact_error: ArtifactError | None = None try: if graceful_signal is not None: process_group_status = signal_group( child.pid, graceful_signal ) - audit.emit( - "process_group_signal", - **_state_fields( - snapshot, - peak_used_bytes, - child, - child.poll(), - process_group_status, - reason, - ), - signal=signal.Signals(graceful_signal).name, - ) + try: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + process_group_status, + reason, + ), + signal=signal.Signals(graceful_signal).name, + ) + except ArtifactError as exc: + artifact_error = exc + audit.disable_component(exc.component) deadline = monotonic() + grace_seconds while monotonic() < deadline: child.poll() @@ -1105,18 +1687,23 @@ def _graceful_cleanup( if escalation_result is not None else reason ) - audit.emit( - "process_group_signal", - **_state_fields( - snapshot, - peak_used_bytes, - child, - child.poll(), - process_group_status, - signal_reason, - ), - signal="SIGKILL", - ) + try: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + process_group_status, + signal_reason, + ), + signal="SIGKILL", + ) + except ArtifactError as exc: + if artifact_error is None: + artifact_error = exc + audit.disable_component(exc.component) except ProcessGroupError as exc: return _emit_final( audit, @@ -1151,6 +1738,10 @@ def _graceful_cleanup( if escalated and escalation_result is not None: classification, exit_code, reason = escalation_result + if artifact_error is not None: + classification = "lease_error" + exit_code = EXIT_LEASE_ERROR + error = f"{artifact_error.component}: {artifact_error}" return _emit_final( audit, classification, @@ -1173,6 +1764,7 @@ def _monitor_child( state: RuntimeState, signal_group: Callable[[int, int], str], group_alive: Callable[[int], bool], + pulse_guardian: Callable[[], None], monotonic: Callable[[], float], sleeper: Callable[[float], None], ) -> int: @@ -1273,19 +1865,6 @@ def _monitor_child( state.peak_used_bytes = max( state.peak_used_bytes, state.snapshot.used_bytes ) - sample_record = audit.emit( - "sample", - **_state_fields( - state.snapshot, - state.peak_used_bytes, - child, - None, - "active", - "none", - ), - ) - audit.heartbeat(sample_record) - if state.snapshot.active_swaps: return _kill_and_finish( audit, @@ -1308,6 +1887,7 @@ def _monitor_child( "used_bytes >= emergency_bytes", signal_group, ) + soft_signal_fields: dict[str, object] | None = None if ( soft_deadline is None and state.snapshot.used_bytes >= config.soft_bytes @@ -1328,8 +1908,7 @@ def _monitor_child( str(exc), ) soft_deadline = now + config.grace_seconds - audit.emit( - "process_group_signal", + soft_signal_fields = { **_state_fields( state.snapshot, state.peak_used_bytes, @@ -1338,9 +1917,39 @@ def _monitor_child( group_status, "used_bytes >= soft_bytes", ), - signal="SIGTERM", - grace_deadline_monotonic=soft_deadline, + "signal": "SIGTERM", + "grace_deadline_monotonic": soft_deadline, + } + try: + pulse_guardian() + except ProcessGroupError as exc: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "signal_error", + EXIT_SIGNAL_ERROR, + str(exc), + signal_group, + ) + if soft_signal_fields is not None: + audit.emit( + "process_group_signal", + **soft_signal_fields, ) + sample_record = audit.emit( + "sample", + **_state_fields( + state.snapshot, + state.peak_used_bytes, + child, + None, + "active", + "none", + ) + ) + audit.heartbeat(sample_record) sleep_seconds = config.sample_interval_seconds if soft_deadline is not None: @@ -1363,6 +1972,9 @@ def run_watchdog( sleeper: Callable[[float], None] | None = None, ) -> int: artifact_paths = config.validate() + use_guardian = ( + launcher is None and sys.platform.startswith("linux") + ) reader = reader or ProcfsReader(config.procfs_root) audit = audit or AuditLogger(sys.stderr) launcher = launcher or subprocess.Popen @@ -1472,8 +2084,8 @@ def restore_child_signal_mask() -> None: signal.pthread_sigmask(signal.SIG_SETMASK, launch_mask) try: + child_environment = os.environ.copy() if lease_manager is not None: - child_environment = os.environ.copy() child_environment.update( { "STRIX_MEMORY_WATCHDOG_LEASE_PATH": str( @@ -1490,6 +2102,14 @@ def restore_child_signal_mask() -> None: ), } ) + if use_guardian: + child = _launch_guardian( + config.command, + child_environment, + config.heartbeat_max_age_seconds, + launch_mask, + ) + elif lease_manager is not None: child = launcher( config.command, start_new_session=True, @@ -1502,7 +2122,7 @@ def restore_child_signal_mask() -> None: start_new_session=True, preexec_fn=restore_child_signal_mask, ) - except (OSError, ValueError) as exc: + except (OSError, ValueError, subprocess.SubprocessError) as exc: detail = getattr(exc, "strerror", None) or str(exc) return _emit_final( audit, @@ -1518,7 +2138,7 @@ def restore_child_signal_mask() -> None: _raise_parent_signal ) if lease_manager is not None: - lease_manager.start(child) + lease_manager.start(child, audit) audit.lease_manager = lease_manager signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) mask_restored = True @@ -1542,6 +2162,7 @@ def restore_child_signal_mask() -> None: state, signal_group, group_alive, + child.pulse if isinstance(child, GuardianProcess) else lambda: None, monotonic, sleeper, ) @@ -1631,6 +2252,8 @@ def restore_child_signal_mask() -> None: signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) if previous_handlers: _restore_parent_signal_handlers(previous_handlers) + if isinstance(child, GuardianProcess): + child.close() def _positive_int(value: str) -> int: @@ -1744,7 +2367,20 @@ def parse_args(argv: Sequence[str]) -> WatchdogConfig: def main(argv: Sequence[str] | None = None) -> int: - config = parse_args(argv if argv is not None else sys.argv[1:]) + arguments = tuple(argv if argv is not None else sys.argv[1:]) + if arguments and arguments[0] == "--internal-guardian": + if len(arguments) < 6 or arguments[4] != "--": + return EXIT_LAUNCH_ERROR + try: + return _guardian_main( + int(arguments[1]), + int(arguments[2]), + _positive_float(arguments[3]), + tuple(arguments[5:]), + ) + except (OSError, ValueError): + return EXIT_LAUNCH_ERROR + config = parse_args(arguments) audit = AuditLogger(sys.stderr) try: return run_watchdog(config, audit=audit) diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index 81c10f84c38f..a61078efb2a3 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import fcntl import hashlib import io import json @@ -400,6 +401,199 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: "parent_signal", ) + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_pipe_close_kills_group_without_fd_leak(self) -> None: + child_code = ( + "import json,os,subprocess,sys,time\n" + "targets=[]\n" + "for name in os.listdir('/proc/self/fd'):\n" + " try: targets.append(os.readlink('/proc/self/fd/'+name))\n" + " except OSError: pass\n" + "grandchild=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'])\n" + "open(sys.argv[1],'w').write(json.dumps({" + "'child':os.getpid(),'grandchild':grandchild.pid," + "'fds':targets}))\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + state_path = root / "state.json" + guardian = watchdog._launch_guardian( + ( + sys.executable, + "-c", + child_code, + str(state_path), + ), + os.environ.copy(), + 0.5, + signal.pthread_sigmask(signal.SIG_BLOCK, ()), + ) + control_target = os.readlink( + f"/proc/self/fd/{guardian.pulse_fd}" + ) + deadline = time.monotonic() + 5 + while not state_path.exists(): + if time.monotonic() >= deadline: + self.fail("guardian payload did not become ready") + time.sleep(0.01) + state = json.loads(state_path.read_text(encoding="utf-8")) + os.close(guardian.pulse_fd) + guardian.wait(timeout=5) + + self.assertNotIn(control_target, state["fds"]) + for process_id in (state["child"], state["grandchild"]): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_documents_setsid_escape_limit(self) -> None: + child_code = ( + "import os,subprocess,sys,time\n" + "escaped=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'],start_new_session=True)\n" + "open(sys.argv[1],'w').write(str(escaped.pid))\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + state_path = Path(temp_dir) / "escaped-pid" + guardian = watchdog._launch_guardian( + ( + sys.executable, + "-c", + child_code, + str(state_path), + ), + os.environ.copy(), + 0.5, + signal.pthread_sigmask(signal.SIG_BLOCK, ()), + ) + deadline = time.monotonic() + 5 + while not state_path.exists(): + if time.monotonic() >= deadline: + self.fail("escaped payload did not become ready") + time.sleep(0.01) + escaped_pid = int( + state_path.read_text(encoding="utf-8") + ) + os.close(guardian.pulse_fd) + guardian.wait(timeout=5) + self.assertTrue(self._process_is_running(escaped_pid)) + os.kill(escaped_pid, signal.SIGKILL) + deadline = time.monotonic() + 2 + while ( + self._process_is_running(escaped_pid) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(escaped_pid)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guard_kills_group_after_watchdog_loss_or_stall(self) -> None: + child_code = ( + "import importlib.util,os,pathlib,subprocess,sys,time\n" + "script=pathlib.Path(sys.argv[1])\n" + "spec=importlib.util.spec_from_file_location('guard_watchdog',script)\n" + "module=importlib.util.module_from_spec(spec)\n" + "sys.modules[spec.name]=module\n" + "spec.loader.exec_module(module)\n" + "module.start_process_group_lease_guard(" + "script,expected_procfs_root=pathlib.Path(sys.argv[2]))\n" + "grandchild=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'])\n" + "open(sys.argv[3],'w').write(" + "f'{os.getpid()} {grandchild.pid}\\n')\n" + "time.sleep(30)\n" + ) + for mode in ("sigkill", "sigstop"): + with self.subTest(mode=mode): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_path = root / "pids" + self._write_procfs_fixture(root) + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--heartbeat-max-age-seconds", + "0.3", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(SCRIPT_PATH), + str(root), + str(pid_path), + ], + stderr=subprocess.PIPE, + text=True, + ) + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if wrapper.poll() is not None: + assert wrapper.stderr is not None + self.fail(wrapper.stderr.read()) + if time.monotonic() >= deadline: + self.fail( + "guarded payload did not become ready" + ) + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + if mode == "sigkill": + wrapper.kill() + else: + os.kill(wrapper.pid, signal.SIGSTOP) + heartbeat_path = root / "heartbeat.json" + heartbeat = json.loads( + heartbeat_path.read_text(encoding="utf-8") + ) + heartbeat["updated_monotonic_ns"] = ( + time.monotonic_ns() + ) + watchdog._write_json_atomic( + heartbeat_path, heartbeat + ) + time.sleep(0.7) + os.kill(wrapper.pid, signal.SIGCONT) + wrapper.wait(timeout=5) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse( + self._process_is_running(process_id) + ) + if wrapper.stderr is not None: + wrapper.stderr.close() + def test_child_sigterm_handler_exits_without_escalation(self) -> None: child_code = ( "import os,signal,sys,time\n" @@ -713,6 +907,89 @@ def write(self, value: str) -> int: audit.final_exit_code, watchdog.EXIT_LEASE_ERROR ) + def test_emergency_signal_precedes_artifact_write(self) -> None: + events: list[str] = [] + + class BlockingAudit(watchdog.AuditLogger): + def __init__(self) -> None: + super().__init__(io.StringIO()) + self.calls = 0 + + def emit( + self, event: str, **fields: object + ) -> dict[str, object]: + self.calls += 1 + events.append(f"audit:{event}") + if self.calls == 3: + raise watchdog.ArtifactError( + "audit", "simulated blocked fsync" + ) + return super().emit(event, **fields) + + def exit_on_kill( + process: FakeProcess, signal_number: int + ) -> None: + events.append(f"signal:{signal_number}") + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), snapshot(160)], + FakeProcess(), + signal_handler=exit_on_kill, + ) + harness.audit = BlockingAudit() + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(events[2], f"signal:{signal.SIGKILL}") + self.assertEqual(events[3], "audit:process_group_signal") + self.assertEqual(harness.process.returncode, -signal.SIGKILL) + + def test_cleanup_reaps_after_persistent_audit_failure(self) -> None: + class FailingSignalAudit(watchdog.AuditLogger): + def emit( + self, event: str, **fields: object + ) -> dict[str, object]: + if event == "process_group_signal": + raise watchdog.ArtifactError( + "audit", "simulated persistent write failure" + ) + return super().emit(event, **fields) + + process = FakeProcess() + signals: list[int] = [] + clock = FakeClock() + + def signal_group( + process_group_id: int, signal_number: int + ) -> str: + signals.append(signal_number) + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + return f"{signal.Signals(signal_number).name.lower()}_sent" + + result = watchdog._graceful_cleanup( + FailingSignalAudit(io.StringIO()), + process, + snapshot(50), + 50, + "parent_signal", + 128 + signal.SIGTERM, + "wrapper received SIGTERM", + signal.SIGTERM, + 0.1, + signal_group, + lambda _process_group_id: process.returncode is None, + clock.monotonic, + clock.sleep, + ) + + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(signals, [signal.SIGTERM, signal.SIGKILL]) + self.assertEqual(process.returncode, -signal.SIGKILL) + def test_invalid_artifact_path_emits_configuration_final(self) -> None: result = subprocess.run( [ @@ -881,18 +1158,36 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: root = Path(temp_dir) process_root = root / "proc" watchdog_pid = 1200 + guardian_pid = 1250 child_pid = 1300 current_pid = 1400 watchdog_start_ticks = 456789 script_path = root / "watchdog.py" script_path.write_text("print('watchdog')\n", encoding="utf-8") - cmdline = ( - b"/usr/bin/python3\0watchdog.py\0--lease-path\0" - ) + lease_path = root / "lease.json" + heartbeat_path = root / "heartbeat.json" + audit_path = root / "audit.jsonl" + command = [sys.executable, "run_matrix.py"] + argv = [ + sys.executable, + "watchdog.py", + "--procfs-root", + "/proc", + "--lease-path", + str(lease_path), + "--heartbeat-path", + str(heartbeat_path), + "--audit-path", + str(audit_path), + "--", + *command, + ] + cmdline = b"\0".join(os.fsencode(value) for value in argv) for process_id, parent_id, group_id, start_ticks in ( (watchdog_pid, 1, watchdog_pid, watchdog_start_ticks), - (child_pid, watchdog_pid, child_pid, 456790), - (current_pid, child_pid, child_pid, 456791), + (guardian_pid, watchdog_pid, guardian_pid, 456790), + (child_pid, guardian_pid, guardian_pid, 456791), + (current_pid, child_pid, guardian_pid, 456792), ): process_dir = process_root / str(process_id) process_dir.mkdir(parents=True) @@ -908,18 +1203,31 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: (process_root / str(watchdog_pid) / "cwd").symlink_to( root, target_is_directory=True ) + (process_root / str(watchdog_pid) / "exe").symlink_to( + Path(sys.executable).resolve() + ) (process_root / str(watchdog_pid) / "cmdline").write_bytes( cmdline ) - lease_path = root / "lease.json" - heartbeat_path = root / "heartbeat.json" - audit_path = root / "audit.jsonl" - audit_path.write_text( - '{"event":"child_started","timestamp":"2026-01-01T00:00:00Z"}\n', - encoding="utf-8", + audit_line = ( + '{"event":"child_started",' + '"timestamp":"2026-01-01T00:00:00Z"}\n' + ) + audit_descriptor = os.open( + audit_path, + os.O_CREAT | os.O_EXCL | os.O_RDWR, + 0o600, ) - command = ["python3", "run_matrix.py"] + os.write(audit_descriptor, audit_line.encode("utf-8")) + os.fsync(audit_descriptor) + fcntl.flock( + audit_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB + ) + audit_status = os.fstat(audit_descriptor) + fd_root = process_root / str(watchdog_pid) / "fd" + fd_root.mkdir() + (fd_root / "9").symlink_to(audit_path) lease = { "format": watchdog.LEASE_FORMAT, "version": watchdog.LEASE_VERSION, @@ -931,6 +1239,9 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: "watchdog_command_sha256": hashlib.sha256( cmdline ).hexdigest(), + "watchdog_executable_path": str( + Path(sys.executable).resolve() + ), "watchdog_script_path": str(script_path), "watchdog_script_sha256": hashlib.sha256( script_path.read_bytes() @@ -938,8 +1249,9 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: "soft_bytes": watchdog.DEFAULT_SOFT_BYTES, "emergency_bytes": watchdog.DEFAULT_EMERGENCY_BYTES, "strict_ceiling_bytes": watchdog.STRICT_CEILING_BYTES, + "guardian_pid": guardian_pid, "child_pid": child_pid, - "child_process_group_id": child_pid, + "child_process_group_id": guardian_pid, "command": command, "child_command_sha256": watchdog._command_sha256( command @@ -947,6 +1259,11 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: "heartbeat_path": str(heartbeat_path), "max_heartbeat_age_seconds": 5.0, "audit_path": str(audit_path), + "audit_device": audit_status.st_dev, + "audit_inode": audit_status.st_ino, + "audit_uid": audit_status.st_uid, + "audit_mode": 0o600, + "audit_fd": 9, "procfs_root": "/proc", } heartbeat = { @@ -962,115 +1279,262 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: watchdog_start_ticks ), "child_pid": child_pid, - "child_process_group_id": child_pid, + "child_process_group_id": guardian_pid, + "sample": { + "audit_record_sha256": hashlib.sha256( + audit_line.encode("utf-8") + ).hexdigest() + }, } - lease_path.write_text(json.dumps(lease), encoding="utf-8") - heartbeat_path.write_text( - json.dumps(heartbeat), encoding="utf-8" + watchdog._write_json_atomic( + lease_path, lease, create=True ) - - validated = watchdog.validate_active_lease( - lease_path, - expected_script_path=script_path, - expected_command_sha256=watchdog._command_sha256( - command - ), - expected_heartbeat_path=heartbeat_path, - expected_audit_path=audit_path, - expected_max_heartbeat_age_seconds=5.0, - current_process_id=current_pid, - process_procfs_root=process_root, - monotonic_ns=lambda: 10_000_000_000, + watchdog._write_json_atomic( + heartbeat_path, heartbeat, create=True ) - self.assertEqual(validated["lease_id"], "test-lease") - with self.subTest("tampered script SHA"): - tampered = dict(lease) - tampered["watchdog_script_sha256"] = "0" * 64 - lease_path.write_text( - json.dumps(tampered), encoding="utf-8" + try: + validation_args = { + "expected_script_path": script_path, + "expected_executable_path": Path(sys.executable), + "expected_command": command, + "expected_heartbeat_path": heartbeat_path, + "expected_audit_path": audit_path, + "expected_max_heartbeat_age_seconds": 5.0, + "current_process_id": current_pid, + "process_procfs_root": process_root, + "monotonic_ns": lambda: 10_000_000_000, + "pidfd_open": lambda _pid: os.open( + os.devnull, os.O_RDONLY + ), + } + validated = watchdog.validate_active_lease( + lease_path, **validation_args ) - with self.assertRaisesRegex( - watchdog.LeaseValidationError, "script SHA" + self.assertEqual(validated["lease_id"], "test-lease") + + def publish_lease( + value: dict[str, Any], + process_argv: list[str] = argv, + ) -> None: + process_cmdline = b"\0".join( + os.fsencode(argument) + for argument in process_argv + ) + (process_root / str(watchdog_pid) / "cmdline").write_bytes( + process_cmdline + ) + value["watchdog_command_sha256"] = hashlib.sha256( + process_cmdline + ).hexdigest() + watchdog._write_json_atomic(lease_path, value) + + for name, bad_argv in ( + ( + "helper inert argument", + [ + sys.executable, + "helper.py", + str(script_path), + *argv[2:], + ], + ), + ( + "python command string", + [ + sys.executable, + "-c", + "pass", + str(script_path), + *argv[2:], + ], + ), + ( + "python module", + [ + sys.executable, + "-m", + "helper", + str(script_path), + *argv[2:], + ], + ), + ( + "interpreter option before script", + [ + sys.executable, + "-O", + str(script_path), + *argv[2:], + ], + ), ): - watchdog.validate_active_lease( - lease_path, - expected_script_path=script_path, - expected_heartbeat_path=heartbeat_path, - expected_audit_path=audit_path, - current_process_id=current_pid, - process_procfs_root=process_root, - monotonic_ns=lambda: 10_000_000_000, + with self.subTest(name): + publish_lease(dict(lease), bad_argv) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "executable argv position", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("wrong command-line policy"): + bad_argv = list(argv) + procfs_index = bad_argv.index("--procfs-root") + 1 + bad_argv[procfs_index] = "/tmp/not-proc" + publish_lease(dict(lease), bad_argv) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "command-line policy", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("wrong monitored command"): + bad_argv = [*argv[:-1], "other_matrix.py"] + bad_lease = dict(lease) + bad_lease["command"] = [ + sys.executable, + "other_matrix.py", + ] + bad_lease["child_command_sha256"] = ( + watchdog._command_sha256( + bad_lease["command"] + ) ) + publish_lease(bad_lease, bad_argv) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "monitored command", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) - with self.subTest("stale heartbeat"): - lease_path.write_text( - json.dumps(lease), encoding="utf-8" - ) - heartbeat["updated_monotonic_ns"] = 1 - heartbeat_path.write_text( - json.dumps(heartbeat), encoding="utf-8" - ) - with self.assertRaisesRegex( - watchdog.LeaseValidationError, "heartbeat is stale" - ): - watchdog.validate_active_lease( - lease_path, - expected_script_path=script_path, - expected_heartbeat_path=heartbeat_path, - expected_audit_path=audit_path, - current_process_id=current_pid, - process_procfs_root=process_root, - monotonic_ns=lambda: 10_000_000_000, + with self.subTest("tampered script SHA"): + tampered = dict(lease) + tampered["watchdog_script_sha256"] = "0" * 64 + publish_lease(tampered) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, "script SHA" + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("stale heartbeat"): + publish_lease(dict(lease)) + heartbeat["updated_monotonic_ns"] = 1 + watchdog._write_json_atomic( + heartbeat_path, heartbeat ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "heartbeat is stale", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) - with self.subTest("arbitrary heartbeat"): - heartbeat["updated_monotonic_ns"] = 9_000_000_000 - heartbeat["lease_id"] = "helper-lease" - heartbeat_path.write_text( - json.dumps(heartbeat), encoding="utf-8" - ) - with self.assertRaisesRegex( - watchdog.LeaseValidationError, - "heartbeat identity", - ): - watchdog.validate_active_lease( - lease_path, - expected_script_path=script_path, - expected_heartbeat_path=heartbeat_path, - expected_audit_path=audit_path, - current_process_id=current_pid, - process_procfs_root=process_root, - monotonic_ns=lambda: 10_000_000_000, + with self.subTest("arbitrary heartbeat"): + heartbeat["updated_monotonic_ns"] = 9_000_000_000 + heartbeat["lease_id"] = "helper-lease" + watchdog._write_json_atomic( + heartbeat_path, heartbeat ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "heartbeat identity", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) - with self.subTest("outside process group"): - heartbeat["lease_id"] = "test-lease" - heartbeat_path.write_text( - json.dumps(heartbeat), encoding="utf-8" - ) - (process_root / str(current_pid) / "stat").write_text( - self._proc_stat( - current_pid, - child_pid, - 9999, - 456791, - ), - encoding="utf-8", - ) - with self.assertRaisesRegex( - watchdog.LeaseValidationError, - "outside the monitored process group", - ): - watchdog.validate_active_lease( - lease_path, - expected_script_path=script_path, - expected_heartbeat_path=heartbeat_path, - expected_audit_path=audit_path, - current_process_id=current_pid, - process_procfs_root=process_root, - monotonic_ns=lambda: 10_000_000_000, + with self.subTest("outside process group"): + heartbeat["lease_id"] = "test-lease" + watchdog._write_json_atomic( + heartbeat_path, heartbeat ) + (process_root / str(current_pid) / "stat").write_text( + self._proc_stat( + current_pid, + child_pid, + 9999, + 456792, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "outside the monitored process group", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + ( + process_root / str(current_pid) / "stat" + ).write_text( + self._proc_stat( + current_pid, + child_pid, + guardian_pid, + 456792, + ), + encoding="utf-8", + ) + + with self.subTest("environment path mismatch"): + bad_validation_args = { + **validation_args, + "expected_heartbeat_path": root / "other.json", + } + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "artifact paths|heartbeat path", + ): + watchdog.validate_active_lease( + lease_path, **bad_validation_args + ) + + with self.subTest("lease inode mismatch"): + publish_lease(dict(lease)) + lease_record = json.loads( + lease_path.read_text(encoding="utf-8") + ) + lease_record["file_inode"] = 0 + lease_path.write_text( + json.dumps(lease_record), encoding="utf-8" + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "identity does not match", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("watchdog start tick mismatch"): + publish_lease(dict(lease)) + (process_root / str(watchdog_pid) / "stat").write_text( + self._proc_stat( + watchdog_pid, + 1, + watchdog_pid, + watchdog_start_ticks + 1, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "start time", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + finally: + os.close(audit_descriptor) def test_zero_swap_gate_launches_and_propagates_child_exit(self) -> None: harness = Harness([snapshot(50)], FakeProcess(returncode=37)) From 0071e8f21701b74b3cc4dfe5c1131950ce338143 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 23:24:49 -0700 Subject: [PATCH 15/56] scripts : harden watchdog cleanup lease Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 14 +- scripts/strix_memory_watchdog.py | 146 +++++++-- tests/test_strix_memory_watchdog.py | 450 ++++++++++++++++++++++++++++ 3 files changed, 579 insertions(+), 31 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 49a1c05547fc..90f80bff7e01 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -21,7 +21,7 @@ The wrapper performs these checks and actions: The 118 GiB emergency threshold leaves a 2 GiB sampling margin below the strict 120 GiB ceiling. The default sample interval is one second. This margin cannot guarantee the ceiling for a workload that can allocate more than 2 GiB between samples. Lower `--emergency-gib` or shorten `--sample-interval-seconds` for such a workload. -Use `--procfs-root` to select a different procfs mount or a test fixture. `--soft-gib`, `--emergency-gib`, `--grace-seconds`, and `--sample-interval-seconds` override the other defaults. The emergency threshold must remain below 120 GiB. +Use `--procfs-root` to select a different procfs mount or a test fixture. `--soft-gib`, `--emergency-gib`, `--grace-seconds`, and `--sample-interval-seconds` override the other defaults. The emergency threshold must remain below 120 GiB. The fail-closed timing bounds are a maximum 30-second grace, maximum one-second sample interval, and maximum five-second heartbeat age. The wrapper writes timestamped JSON Lines records to standard error. Preflight, sample, signal, and final records include total, available, used, and peak-used bytes, swap entry count, child status, process-group status, threshold reason, and final classification where applicable. Signal records are written immediately after each process-group signal. Child standard input, standard output, and standard error are inherited unchanged. @@ -40,28 +40,28 @@ Use all three artifact options together when another process must prove that it The watchdog creates and exclusively locks the persistent audit before launch. It then starts an internal guardian as the new session and process-group leader; the guardian starts the supplied command in that same group without inheriting the private control pipe. After the guardian reports the payload PID, the watchdog atomically creates the lease and heartbeat. Existing artifact paths are rejected rather than overwritten. The payload receives the resolved paths through `STRIX_MEMORY_WATCHDOG_LEASE_PATH`, `STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH`, and `STRIX_MEMORY_WATCHDOG_AUDIT_PATH`. It also receives `STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS`. -The child can run before the first atomic lease rename. A matching preflight must retry the inherited lease path for a bounded interval and fail closed if a complete valid lease does not appear. It must not accept a lease path supplied separately by the operator. +The child can run before the first atomic lease rename. A matching preflight must retry the inherited lease path for a bounded interval and fail closed if a complete valid lease does not appear. It must not accept a lease path supplied separately by the operator. Consumers must require version 2; version 1 does not describe the guardian topology or timing policy and is rejected. -Lease format `strix-memory-watchdog-lease`, version 1, contains: +Lease format `strix-memory-watchdog-lease`, version 2, contains: - `lease_id` and active/final `state` - `watchdog_pid`, `watchdog_start_time_utc`, Linux `watchdog_start_time_ticks`, `watchdog_executable_path`, `watchdog_command_sha256`, `watchdog_script_path`, and `watchdog_script_sha256` -- exact `soft_bytes`, `emergency_bytes`, and `strict_ceiling_bytes` +- exact `soft_bytes`, `emergency_bytes`, `strict_ceiling_bytes`, `grace_seconds`, and `sample_interval_seconds` - `procfs_root` - `guardian_pid`, payload `child_pid`, `child_process_group_id`, `command`, and `child_command_sha256` - `heartbeat_path`, `max_heartbeat_age_seconds`, and `audit_path` - device, inode, owner, and mode identity for atomic JSON artifacts, plus the watchdog-held audit descriptor identity - the authoritative `final` audit record after termination -Heartbeat format `strix-memory-watchdog-heartbeat`, version 1, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample first checks swap and memory thresholds, pulses the guardian through the private nonblocking pipe, then atomically replaces the heartbeat with the complete sample audit record and its persistent-audit record hash. A blocked audit or heartbeat write cannot delay the emergency signal. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. +Heartbeat format `strix-memory-watchdog-heartbeat`, version 2, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample first checks swap and memory thresholds, pulses the guardian through the private nonblocking pipe, then atomically replaces the heartbeat with the complete sample audit record and its persistent-audit record hash. It pulses again after persistence succeeds. A blocked audit or heartbeat write cannot delay the emergency signal; if persistence stalls past the guardian deadline, the guardian fails closed. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. -The guardian uses Linux `PR_SET_PDEATHSIG` with a parent-race check. It kills its process group on watchdog death, control-pipe EOF/error, or a missed pulse deadline, including a stopped or wedged watchdog. The payload must call `start_process_group_lease_guard()` before it starts exporter descendants. This validates the lease with bounded startup retries, arms a second parent-death link to the guardian, and starts a thread that kills the process group if the watchdog evidence becomes stale or invalid. +The guardian uses Linux `PR_SET_PDEATHSIG` with a parent-race check. It kills its process group on watchdog death, control-pipe EOF/error, or a missed pulse deadline, including a stopped or wedged watchdog. When the watchdog sends a graceful signal, it also puts the guardian into a bounded grace mode and continues private pulses while it waits. This lets the watchdog own the configured grace deadline and record any `SIGKILL` escalation instead of letting the shorter heartbeat deadline preempt cleanup. The payload must call `start_process_group_lease_guard()` before it starts exporter descendants. This validates the lease with bounded startup retries, arms a second parent-death link to the guardian, and starts a thread that kills the process group if any validation or artifact operation fails or the watchdog evidence becomes stale. A matching Linux preflight must verify all of the following: - The inherited lease, heartbeat, and audit paths match the paths inside the lease. - `/proc//exe` is the exact expected Python executable and argv position 1 is the exact repository watchdog script. `-c`, `-m`, helper-script, inert-argument, and interpreter-option substitutions are rejected. -- The watchdog command line itself supplies the exact 116/118 GiB thresholds, `/proc`, inherited artifact paths, and command after `--`; the lease cannot override those expectations. +- The watchdog command line itself supplies the exact 116/118 GiB thresholds, `/proc`, inherited artifact paths, timing policy, and command after `--`; the lease cannot override those expectations. - `/proc//stat` start ticks and `/proc//cmdline` SHA-256 match the lease and remain stable across validation. A pidfd is held during validation when Linux provides `pidfd_open`. - The topology is watchdog parent -> guardian process-group leader -> payload child. The current process must be inside `child_process_group_id`. - The command identity is expected, the procfs root is `/proc`, and thresholds are exactly 116 GiB soft, 118 GiB emergency, and 120 GiB strict ceiling for the final run. diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index dd185a00f5f2..a16b79743e10 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -32,11 +32,14 @@ DEFAULT_GRACE_SECONDS = 30.0 DEFAULT_SAMPLE_INTERVAL_SECONDS = 1.0 DEFAULT_HEARTBEAT_MAX_AGE_SECONDS = 5.0 +MAX_GRACE_SECONDS = 30.0 +MAX_SAMPLE_INTERVAL_SECONDS = 1.0 +MAX_HEARTBEAT_MAX_AGE_SECONDS = 5.0 LEASE_FORMAT = "strix-memory-watchdog-lease" -LEASE_VERSION = 1 +LEASE_VERSION = 2 HEARTBEAT_FORMAT = "strix-memory-watchdog-heartbeat" -HEARTBEAT_VERSION = 1 +HEARTBEAT_VERSION = 2 PR_SET_PDEATHSIG = 1 LEASE_GUARD_SIGNAL = signal.SIGUSR1 @@ -106,8 +109,14 @@ def wait(self, timeout: float | None = None) -> int: return self.process.wait(timeout=timeout) def pulse(self) -> None: + self._write_control(b"P") + + def begin_grace(self) -> None: + self._write_control(b"G") + + def _write_control(self, value: bytes) -> None: try: - os.write(self.pulse_fd, b"\0") + os.write(self.pulse_fd, value) except BlockingIOError as exc: raise ProcessGroupError( "guardian pulse pipe is blocked" @@ -175,20 +184,32 @@ def validate(self) -> ArtifactPaths | None: raise ValueError("emergency threshold must be greater than soft threshold") if self.emergency_bytes >= STRICT_CEILING_BYTES: raise ValueError("emergency threshold must be below 120 GiB") - if not math.isfinite(self.grace_seconds) or self.grace_seconds <= 0: - raise ValueError("grace period must be greater than zero") + if ( + not math.isfinite(self.grace_seconds) + or self.grace_seconds <= 0 + or self.grace_seconds > MAX_GRACE_SECONDS + ): + raise ValueError( + "grace period must be greater than zero and at most 30 seconds" + ) if ( not math.isfinite(self.sample_interval_seconds) or self.sample_interval_seconds <= 0 + or self.sample_interval_seconds > MAX_SAMPLE_INTERVAL_SECONDS ): - raise ValueError("sample interval must be greater than zero") + raise ValueError( + "sample interval must be greater than zero and at most 1 second" + ) if ( not math.isfinite(self.heartbeat_max_age_seconds) or self.heartbeat_max_age_seconds <= self.sample_interval_seconds + or self.heartbeat_max_age_seconds + > MAX_HEARTBEAT_MAX_AGE_SECONDS ): raise ValueError( - "heartbeat max age must be greater than sample interval" + "heartbeat max age must be greater than sample interval " + "and at most 5 seconds" ) lease_paths = ( self.lease_path, @@ -348,6 +369,7 @@ def _guardian_main( control_fd: int, status_fd: int, pulse_timeout_seconds: float, + grace_timeout_seconds: float, command: tuple[str, ...], ) -> int: if not sys.platform.startswith("linux"): @@ -386,6 +408,7 @@ def prepare_payload() -> None: control_fd, select.POLLIN | select.POLLHUP | select.POLLERR, ) + current_timeout_seconds = pulse_timeout_seconds deadline = time.monotonic() + pulse_timeout_seconds while True: remaining = max(0.0, deadline - time.monotonic()) @@ -399,16 +422,20 @@ def prepare_payload() -> None: pulse = b"" if not pulse: _kill_own_process_group() - deadline = time.monotonic() + pulse_timeout_seconds + if b"G" in pulse: + current_timeout_seconds = grace_timeout_seconds + deadline = time.monotonic() + current_timeout_seconds if time.monotonic() >= deadline: _kill_own_process_group() returncode = payload.poll() if returncode is not None: - return ( - 128 - returncode - if returncode < 0 - else returncode - ) + if returncode >= 0: + return returncode + signal_number = -returncode + if signal_number not in (signal.SIGKILL, signal.SIGSTOP): + signal.signal(signal_number, signal.SIG_DFL) + os.kill(os.getpid(), signal_number) + return 128 + signal_number def _read_guardian_status( @@ -450,6 +477,7 @@ def _launch_guardian( command: tuple[str, ...], environment: dict[str, str], pulse_timeout_seconds: float, + grace_timeout_seconds: float, launch_mask: set[signal.Signals], ) -> GuardianProcess: control_read, control_write = os.pipe() @@ -469,6 +497,7 @@ def prepare_guardian() -> None: str(control_read), str(status_write), str(pulse_timeout_seconds), + str(grace_timeout_seconds), "--", *command, ) @@ -712,6 +741,8 @@ def start( "soft_bytes": self.config.soft_bytes, "emergency_bytes": self.config.emergency_bytes, "strict_ceiling_bytes": STRICT_CEILING_BYTES, + "grace_seconds": self.config.grace_seconds, + "sample_interval_seconds": self.config.sample_interval_seconds, "guardian_pid": child.pid, "child_pid": payload_pid, "child_process_group_id": child.pid, @@ -956,6 +987,16 @@ def validate_active_lease( raise LeaseValidationError( "watchdog command-line policy does not match" ) + if ( + lease.get("grace_seconds") != live_config.grace_seconds + or lease.get("sample_interval_seconds") + != live_config.sample_interval_seconds + or lease.get("max_heartbeat_age_seconds") + != live_config.heartbeat_max_age_seconds + ): + raise LeaseValidationError( + "watchdog lease timing policy does not match" + ) if ( live_paths is None or live_paths.lease != lease_path @@ -1246,7 +1287,7 @@ def start_process_group_lease_guard( process_procfs_root=process_procfs_root, ) break - except LeaseValidationError: + except Exception: if time.monotonic() >= deadline: raise time.sleep(0.01) @@ -1272,7 +1313,7 @@ def monitor() -> None: expected_max_heartbeat_age_seconds=max_age_seconds, process_procfs_root=process_procfs_root, ) - except LeaseValidationError: + except Exception: _kill_own_process_group() guard = threading.Thread( @@ -1291,6 +1332,7 @@ def __init__( wall_clock: Callable[[], datetime] | None = None, ): self.stream = stream + self.stream_enabled = True self.wall_clock = wall_clock self.persistent_stream: IO[str] | None = None self.lease_manager: LeaseManager | None = None @@ -1337,15 +1379,21 @@ def persistent_identity(self) -> dict[str, int]: } def close(self) -> None: - if self.persistent_stream is not None: - self.persistent_stream.close() - self.persistent_stream = None + persistent_stream = self.persistent_stream + self.persistent_stream = None + if persistent_stream is not None: + try: + persistent_stream.close() + except (OSError, ValueError): + pass def disable_component(self, component: str) -> None: if component == "audit": self.close() elif component == "lease": self.lease_manager = None + elif component == "stderr": + self.stream_enabled = False def emit(self, event: str, **fields: object) -> dict[str, object]: record = { @@ -1357,8 +1405,16 @@ def emit(self, event: str, **fields: object) -> dict[str, object]: json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" ) - self.stream.write(line) - self.stream.flush() + if self.stream_enabled: + try: + self.stream.write(line) + self.stream.flush() + except (OSError, ValueError) as exc: + detail = getattr(exc, "strerror", None) or str(exc) + raise ArtifactError( + "stderr", + f"cannot write standard error audit: {detail}", + ) from exc self.last_record_sha256 = _sha256_bytes(line.encode("utf-8")) if self.persistent_stream is not None: try: @@ -1654,6 +1710,11 @@ def _graceful_cleanup( process_group_status = signal_group( child.pid, graceful_signal ) + if ( + isinstance(child, GuardianProcess) + and child.poll() is None + ): + child.begin_grace() try: audit.emit( "process_group_signal", @@ -1675,6 +1736,11 @@ def _graceful_cleanup( child.poll() if not group_alive(child.pid): break + if ( + isinstance(child, GuardianProcess) + and child.poll() is None + ): + child.pulse() sleeper(min(0.05, deadline - monotonic())) child.poll() if group_alive(child.pid): @@ -1908,6 +1974,20 @@ def _monitor_child( str(exc), ) soft_deadline = now + config.grace_seconds + if isinstance(child, GuardianProcess): + try: + child.begin_grace() + except ProcessGroupError as exc: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "signal_error", + EXIT_SIGNAL_ERROR, + str(exc), + signal_group, + ) soft_signal_fields = { **_state_fields( state.snapshot, @@ -1923,6 +2003,8 @@ def _monitor_child( try: pulse_guardian() except ProcessGroupError as exc: + if child.poll() is not None: + continue return _kill_and_finish( audit, child, @@ -1950,6 +2032,21 @@ def _monitor_child( ) ) audit.heartbeat(sample_record) + try: + pulse_guardian() + except ProcessGroupError as exc: + if child.poll() is not None: + continue + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "signal_error", + EXIT_SIGNAL_ERROR, + str(exc), + signal_group, + ) sleep_seconds = config.sample_interval_seconds if soft_deadline is not None: @@ -2107,6 +2204,7 @@ def restore_child_signal_mask() -> None: config.command, child_environment, config.heartbeat_max_age_seconds, + config.grace_seconds + 1.0, launch_mask, ) elif lease_manager is not None: @@ -2168,8 +2266,7 @@ def restore_child_signal_mask() -> None: ) except ArtifactError as exc: _set_parent_signal_handlers(signal.SIG_IGN) - if exc.component == "audit": - audit.disable_component(exc.component) + audit.disable_component(exc.component) if child is None: return _emit_final( audit, @@ -2369,14 +2466,15 @@ def parse_args(argv: Sequence[str]) -> WatchdogConfig: def main(argv: Sequence[str] | None = None) -> int: arguments = tuple(argv if argv is not None else sys.argv[1:]) if arguments and arguments[0] == "--internal-guardian": - if len(arguments) < 6 or arguments[4] != "--": + if len(arguments) < 7 or arguments[5] != "--": return EXIT_LAUNCH_ERROR try: return _guardian_main( int(arguments[1]), int(arguments[2]), _positive_float(arguments[3]), - tuple(arguments[5:]), + _positive_float(arguments[4]), + tuple(arguments[6:]), ) except (OSError, ValueError): return EXIT_LAUNCH_ERROR diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index a61078efb2a3..3a0a534717da 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -401,6 +401,168 @@ def test_parent_signals_leave_no_child_or_grandchild(self) -> None: "parent_signal", ) + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_parent_signal_grace_outlives_guardian_pulse_timeout( + self, + ) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "open(sys.argv[1],'w').write(str(os.getpid()));" + "time.sleep(30)" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pid" + stderr_path = root / "stderr.jsonl" + self._write_procfs_fixture(root) + with stderr_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.4", + "--sample-interval-seconds", + "0.05", + "--heartbeat-max-age-seconds", + "0.1", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + child_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + self.fail("child process did not become ready") + time.sleep(0.01) + child_pid = int( + pid_file.read_text(encoding="utf-8") + ) + started = time.monotonic() + wrapper.send_signal(signal.SIGTERM) + wrapper.wait(timeout=5) + elapsed = time.monotonic() - started + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in stderr_path.read_text( + encoding="utf-8" + ).splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertGreaterEqual(elapsed, 0.35) + self.assertEqual( + wrapper.returncode, + 128 + signal.SIGTERM, + records, + ) + self.assertEqual(signals, ["SIGTERM", "SIGKILL"]) + self.assertEqual( + records[-1]["classification"], "parent_signal" + ) + self.assertFalse(self._process_is_running(child_pid)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_parent_signal_allows_exit_after_pulse_deadline(self) -> None: + child_code = ( + "import os,signal,sys,time\n" + "def stop(_signal,_frame):\n" + " time.sleep(0.25)\n" + " raise SystemExit(0)\n" + "signal.signal(signal.SIGTERM,stop)\n" + "open(sys.argv[1],'w').write(str(os.getpid()))\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pid" + stderr_path = root / "stderr.jsonl" + self._write_procfs_fixture(root) + with stderr_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.4", + "--sample-interval-seconds", + "0.05", + "--heartbeat-max-age-seconds", + "0.1", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + try: + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + self.fail("child process did not become ready") + time.sleep(0.01) + started = time.monotonic() + wrapper.send_signal(signal.SIGTERM) + wrapper.wait(timeout=5) + elapsed = time.monotonic() - started + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + + records = [ + json.loads(line) + for line in stderr_path.read_text( + encoding="utf-8" + ).splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertGreaterEqual(elapsed, 0.2) + self.assertLess(elapsed, 0.4) + self.assertEqual(wrapper.returncode, 128 + signal.SIGTERM) + self.assertEqual(signals, ["SIGTERM"]) + self.assertEqual(records[-1]["child_returncode"], 0) + @unittest.skipUnless( sys.platform.startswith("linux"), "Linux guardian lifecycle", @@ -431,6 +593,7 @@ def test_guardian_pipe_close_kills_group_without_fd_leak(self) -> None: ), os.environ.copy(), 0.5, + 1.0, signal.pthread_sigmask(signal.SIG_BLOCK, ()), ) control_target = os.readlink( @@ -478,6 +641,7 @@ def test_guardian_documents_setsid_escape_limit(self) -> None: ), os.environ.copy(), 0.5, + 1.0, signal.pthread_sigmask(signal.SIG_BLOCK, ()), ) deadline = time.monotonic() + 5 @@ -594,6 +758,96 @@ def test_guard_kills_group_after_watchdog_loss_or_stall(self) -> None: if wrapper.stderr is not None: wrapper.stderr.close() + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_payload_guard_fails_closed_on_artifact_error(self) -> None: + child_code = ( + "import importlib.util,os,pathlib,subprocess,sys,time\n" + "script=pathlib.Path(sys.argv[1])\n" + "spec=importlib.util.spec_from_file_location('guard_watchdog',script)\n" + "module=importlib.util.module_from_spec(spec)\n" + "sys.modules[spec.name]=module\n" + "spec.loader.exec_module(module)\n" + "module.start_process_group_lease_guard(" + "script,expected_procfs_root=pathlib.Path(sys.argv[2]))\n" + "def fail(*_args,**_kwargs):\n" + " raise module.ArtifactError('script','unreadable')\n" + "module.validate_active_lease=fail\n" + "grandchild=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'])\n" + "open(sys.argv[3],'w').write(" + "f'{os.getpid()} {grandchild.pid}\\n')\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_path = root / "pids" + self._write_procfs_fixture(root) + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--heartbeat-max-age-seconds", + "0.3", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(SCRIPT_PATH), + str(root), + str(pid_path), + ], + stderr=subprocess.PIPE, + text=True, + ) + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if wrapper.poll() is not None: + assert wrapper.stderr is not None + self.fail(wrapper.stderr.read()) + if time.monotonic() >= deadline: + self.fail("guarded payload did not become ready") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + if wrapper.stderr is not None: + wrapper.stderr.close() + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + assert child_pid is not None + assert grandchild_pid is not None + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + def test_child_sigterm_handler_exits_without_escalation(self) -> None: child_code = ( "import os,signal,sys,time\n" @@ -869,6 +1123,143 @@ def test_configuration_rejects_non_finite_timing(self) -> None: with self.assertRaisesRegex(ValueError, "grace period"): config.validate() + def test_configuration_rejects_weakened_liveness_timing(self) -> None: + cases = ( + ( + {"grace_seconds": 31.0}, + "grace period", + ), + ( + {"sample_interval_seconds": 1.1}, + "sample interval", + ), + ( + { + "sample_interval_seconds": 1.0, + "heartbeat_max_age_seconds": 5.1, + }, + "heartbeat max age", + ), + ) + for overrides, message in cases: + with self.subTest(overrides=overrides): + config = watchdog.WatchdogConfig( + command=("fake-command",), + **overrides, + ) + with self.assertRaisesRegex(ValueError, message): + config.validate() + + def test_stderr_failure_does_not_bypass_cleanup(self) -> None: + class FailingStderr(io.StringIO): + def write(self, value: str) -> int: + raise OSError("stderr closed") + + process = FakeProcess() + + def exit_on_kill( + target: FakeProcess, signal_number: int + ) -> None: + if signal_number == signal.SIGKILL: + target.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50)], + process, + signal_handler=exit_on_kill, + ) + harness.audit = watchdog.AuditLogger(FailingStderr()) + with tempfile.TemporaryDirectory() as temp_dir: + persistent_path = Path(temp_dir) / "audit.jsonl" + harness.audit.open_persistent(persistent_path) + result = watchdog._graceful_cleanup( + harness.audit, + process, + snapshot(50), + 50, + "internal_error", + watchdog.EXIT_INTERNAL_ERROR, + "test cleanup", + signal.SIGTERM, + 0.1, + harness.signal_group, + harness.group_alive, + harness.clock.monotonic, + harness.clock.sleep, + ) + harness.audit.close() + records = [ + json.loads(line) + for line in persistent_path.read_text( + encoding="utf-8" + ).splitlines() + ] + + self.assertEqual( + harness.signals, [signal.SIGTERM, signal.SIGKILL] + ) + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(records[-1]["classification"], "lease_error") + + def test_audit_write_and_close_failures_do_not_bypass_cleanup( + self, + ) -> None: + class FailingPersistent(io.StringIO): + def __init__(self) -> None: + super().__init__() + self.close_called = False + + def write(self, value: str) -> int: + raise OSError("persistent write failed") + + def close(self) -> None: + if self.close_called: + super().close() + return + self.close_called = True + raise OSError("persistent close failed") + + process = FakeProcess() + + def exit_on_kill( + target: FakeProcess, signal_number: int + ) -> None: + if signal_number == signal.SIGKILL: + target.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50)], + process, + signal_handler=exit_on_kill, + ) + persistent = FailingPersistent() + harness.audit.persistent_stream = persistent + result = watchdog._graceful_cleanup( + harness.audit, + process, + snapshot(50), + 50, + "internal_error", + watchdog.EXIT_INTERNAL_ERROR, + "test cleanup", + signal.SIGTERM, + 0.1, + harness.signal_group, + harness.group_alive, + harness.clock.monotonic, + harness.clock.sleep, + ) + + self.assertTrue(persistent.close_called) + self.assertEqual( + harness.signals, [signal.SIGTERM, signal.SIGKILL] + ) + self.assertEqual(process.returncode, -signal.SIGKILL) + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual( + harness.records()[-1]["classification"], "lease_error" + ) + def test_final_record_survives_artifact_failures(self) -> None: class FailingLease: def finalize(self, record: dict[str, Any]) -> None: @@ -1114,6 +1505,49 @@ def test_cli_fixture_launches_command_and_propagates_exit(self) -> None: str((root / "persistent-audit.jsonl").resolve()), ) + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_preserves_payload_signal_status(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self._write_procfs_fixture(root) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--sample-interval-seconds", + "0.01", + "--", + sys.executable, + "-c", + ( + "import os,signal,time;" + "time.sleep(0.1);" + "os.kill(os.getpid(),signal.SIGTERM)" + ), + ], + capture_output=True, + check=False, + text=True, + timeout=5, + ) + records = [ + json.loads(line) + for line in result.stderr.splitlines() + ] + + self.assertEqual(result.returncode, 128 + signal.SIGTERM) + self.assertEqual(records[-1]["classification"], "child_exit") + self.assertEqual( + records[-1]["child_returncode"], -signal.SIGTERM + ) + self.assertEqual(records[-1]["child_status"], "signaled") + def test_existing_lease_fails_closed_and_stops_child(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -1249,6 +1683,10 @@ def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: "soft_bytes": watchdog.DEFAULT_SOFT_BYTES, "emergency_bytes": watchdog.DEFAULT_EMERGENCY_BYTES, "strict_ceiling_bytes": watchdog.STRICT_CEILING_BYTES, + "grace_seconds": watchdog.DEFAULT_GRACE_SECONDS, + "sample_interval_seconds": ( + watchdog.DEFAULT_SAMPLE_INTERVAL_SECONDS + ), "guardian_pid": guardian_pid, "child_pid": child_pid, "child_process_group_id": guardian_pid, @@ -1392,6 +1830,18 @@ def publish_lease( lease_path, **validation_args ) + with self.subTest("wrong lease timing policy"): + bad_lease = dict(lease) + bad_lease["grace_seconds"] = 29.0 + publish_lease(bad_lease) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "lease timing policy", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + with self.subTest("wrong monitored command"): bad_argv = [*argv[:-1], "other_matrix.py"] bad_lease = dict(lease) From cd97b50b664bd170819b4225ae52cad97d762ada Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 23:39:59 -0700 Subject: [PATCH 16/56] scripts : enforce cleanup on guardian errors Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 2 +- scripts/strix_memory_watchdog.py | 54 ++++++++---- tests/test_strix_memory_watchdog.py | 123 ++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 16 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index 90f80bff7e01..da3560caddd9 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -55,7 +55,7 @@ Lease format `strix-memory-watchdog-lease`, version 2, contains: Heartbeat format `strix-memory-watchdog-heartbeat`, version 2, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample first checks swap and memory thresholds, pulses the guardian through the private nonblocking pipe, then atomically replaces the heartbeat with the complete sample audit record and its persistent-audit record hash. It pulses again after persistence succeeds. A blocked audit or heartbeat write cannot delay the emergency signal; if persistence stalls past the guardian deadline, the guardian fails closed. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. -The guardian uses Linux `PR_SET_PDEATHSIG` with a parent-race check. It kills its process group on watchdog death, control-pipe EOF/error, or a missed pulse deadline, including a stopped or wedged watchdog. When the watchdog sends a graceful signal, it also puts the guardian into a bounded grace mode and continues private pulses while it waits. This lets the watchdog own the configured grace deadline and record any `SIGKILL` escalation instead of letting the shorter heartbeat deadline preempt cleanup. The payload must call `start_process_group_lease_guard()` before it starts exporter descendants. This validates the lease with bounded startup retries, arms a second parent-death link to the guardian, and starts a thread that kills the process group if any validation or artifact operation fails or the watchdog evidence becomes stale. +The guardian uses Linux `PR_SET_PDEATHSIG` with a parent-race check. It kills its process group on watchdog death, control-pipe EOF/error, or a missed pulse deadline, including a stopped or wedged watchdog. When the watchdog sends a graceful signal, it also puts the guardian into a bounded grace mode and continues private pulses while it waits. This lets the watchdog own the configured grace deadline and record any `SIGKILL` escalation instead of letting the shorter heartbeat deadline preempt cleanup. If the grace control message or a cleanup pulse fails, the watchdog independently sends `SIGKILL` to the process group and reaps the child before it reports `signal_error`. The payload must call `start_process_group_lease_guard()` before it starts exporter descendants. This validates the lease with bounded startup retries, arms a second parent-death link to the guardian, and starts a thread that kills the process group if any validation or artifact operation fails or the watchdog evidence becomes stale. A matching Linux preflight must verify all of the following: diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index a16b79743e10..0eb6df6d78ba 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -1705,16 +1705,12 @@ def _graceful_cleanup( ) -> int: escalated = False artifact_error: ArtifactError | None = None + guardian_control_error: ProcessGroupError | None = None try: if graceful_signal is not None: process_group_status = signal_group( child.pid, graceful_signal ) - if ( - isinstance(child, GuardianProcess) - and child.poll() is None - ): - child.begin_grace() try: audit.emit( "process_group_signal", @@ -1731,8 +1727,19 @@ def _graceful_cleanup( except ArtifactError as exc: artifact_error = exc audit.disable_component(exc.component) + if ( + isinstance(child, GuardianProcess) + and child.poll() is None + ): + try: + child.begin_grace() + except ProcessGroupError as exc: + guardian_control_error = exc deadline = monotonic() + grace_seconds - while monotonic() < deadline: + while ( + guardian_control_error is None + and monotonic() < deadline + ): child.poll() if not group_alive(child.pid): break @@ -1740,19 +1747,31 @@ def _graceful_cleanup( isinstance(child, GuardianProcess) and child.poll() is None ): - child.pulse() + try: + child.pulse() + except ProcessGroupError as exc: + guardian_control_error = exc + break sleeper(min(0.05, deadline - monotonic())) child.poll() - if group_alive(child.pid): + if ( + guardian_control_error is not None + or group_alive(child.pid) + ): escalated = True process_group_status = signal_group( child.pid, signal.SIGKILL ) - signal_reason = ( - escalation_result[2] - if escalation_result is not None - else reason - ) + if guardian_control_error is not None: + signal_reason = ( + "guardian control failed during graceful cleanup" + ) + else: + signal_reason = ( + escalation_result[2] + if escalation_result is not None + else reason + ) try: audit.emit( "process_group_signal", @@ -1802,9 +1821,14 @@ def _graceful_cleanup( str(exc), ) - if escalated and escalation_result is not None: + if guardian_control_error is not None: + classification = "signal_error" + exit_code = EXIT_SIGNAL_ERROR + reason = "guardian control failed during graceful cleanup" + error = str(guardian_control_error) + elif escalated and escalation_result is not None: classification, exit_code, reason = escalation_result - if artifact_error is not None: + if artifact_error is not None and guardian_control_error is None: classification = "lease_error" exit_code = EXIT_LEASE_ERROR error = f"{artifact_error.component}: {artifact_error}" diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index 3a0a534717da..d1cfb7f3ef6a 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -563,6 +563,129 @@ def test_parent_signal_allows_exit_after_pulse_deadline(self) -> None: self.assertEqual(signals, ["SIGTERM"]) self.assertEqual(records[-1]["child_returncode"], 0) + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_control_failure_still_kills_and_reaps_group( + self, + ) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "grandchild=os.fork();" + "\nif grandchild == 0:\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n');" + " time.sleep(30)\n" + ) + for mode in ("closed", "blocked"): + with self.subTest(mode=mode): + with tempfile.TemporaryDirectory() as temp_dir: + pid_path = Path(temp_dir) / "pids" + process = subprocess.Popen( + [ + sys.executable, + "-c", + child_code, + str(pid_path), + ], + start_new_session=True, + ) + read_fd, write_fd = os.pipe() + os.set_blocking(write_fd, False) + if mode == "closed": + os.close(write_fd) + write_fd = -1 + else: + try: + while True: + os.write(write_fd, b"x" * 65536) + except BlockingIOError: + pass + guardian = watchdog.GuardianProcess( + process, + process.pid, + write_fd, + ) + stream = io.StringIO() + audit = watchdog.AuditLogger(stream) + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if time.monotonic() >= deadline: + self.fail( + "child process group did not start" + ) + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + result = watchdog._graceful_cleanup( + audit, + guardian, + snapshot(50), + 50, + "parent_signal", + 128 + signal.SIGTERM, + "wrapper received SIGTERM", + signal.SIGTERM, + 0.4, + watchdog._signal_process_group, + watchdog._process_group_alive, + time.monotonic, + time.sleep, + ) + finally: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + guardian.close() + os.close(read_fd) + + records = [ + json.loads(line) + for line in stream.getvalue().splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual( + result, watchdog.EXIT_SIGNAL_ERROR + ) + self.assertEqual(signals, ["SIGTERM", "SIGKILL"]) + self.assertEqual( + records[-1]["classification"], "signal_error" + ) + self.assertEqual( + records[-1]["child_returncode"], + -signal.SIGKILL, + ) + assert child_pid is not None + assert grandchild_pid is not None + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse( + self._process_is_running(process_id) + ) + @unittest.skipUnless( sys.platform.startswith("linux"), "Linux guardian lifecycle", From f6b4da49b913f7e6fe739e9ca662387f8c2ef659 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 23:50:22 -0700 Subject: [PATCH 17/56] scripts : preserve watchdog failure cause Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/strix-memory-watchdog.md | 2 +- scripts/strix_memory_watchdog.py | 59 +++++-- tests/test_strix_memory_watchdog.py | 259 +++++++++++++++++----------- 3 files changed, 202 insertions(+), 118 deletions(-) diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md index da3560caddd9..f0099b9a55ad 100644 --- a/docs/strix-memory-watchdog.md +++ b/docs/strix-memory-watchdog.md @@ -73,7 +73,7 @@ These checks reject accidental or helper-process substitution and make regular-f The guardian controls only the process group. A payload that deliberately calls `setsid()` can escape it. The correctness harness must not do that. If arbitrary payload code is in scope, launch the watchdog in a service/cgroup configured to kill every member when the unit stops. -Exit classifications are authoritative in the final JSON record. Operational failures use these exit codes: +Exit classifications are authoritative in the last final JSON record. If final artifact persistence fails after a primary safety failure, the primary classification and exit code remain unchanged and the artifact failure is listed in `secondary_errors`. Operational failures use these exit codes: | Exit code | Classification | | ---: | --- | diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index 0eb6df6d78ba..75f3e8852e27 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -1491,6 +1491,7 @@ def _emit_final( child_returncode: int | None = None, process_group_status: str = "not_created", error: str | None = None, + preserve_primary_on_artifact_error: bool = False, ) -> int: fields = _state_fields( snapshot, @@ -1503,31 +1504,52 @@ def _emit_final( fields.update(classification=classification, exit_code=exit_code) if error: fields["error"] = error + + def record_artifact_error(exc: ArtifactError) -> None: + nonlocal exit_code + detail = { + "component": exc.component, + "detail": str(exc), + } + if preserve_primary_on_artifact_error: + secondary_errors = fields.setdefault( + "secondary_errors", [] + ) + assert isinstance(secondary_errors, list) + secondary_errors.append(detail) + else: + fields.update( + classification="lease_error", + exit_code=EXIT_LEASE_ERROR, + threshold_reason="watchdog artifact finalization failed", + error=f"{exc.component}: {exc}", + ) + exit_code = EXIT_LEASE_ERROR + + def emit_final_record() -> dict[str, object]: + try: + return audit.emit("final", **fields) + except ArtifactError as exc: + audit.disable_component(exc.component) + record_artifact_error(exc) + try: + return audit.emit("final", **fields) + except ArtifactError as exc: + audit.disable_component(exc.component) + record_artifact_error(exc) + return audit.emit("final", **fields) + previous_mask = signal.pthread_sigmask( signal.SIG_BLOCK, PARENT_SIGNALS ) try: + record = emit_final_record() try: - record = audit.emit("final", **fields) audit.finalize(record) except ArtifactError as exc: audit.disable_component(exc.component) - fields.update( - classification="lease_error", - exit_code=EXIT_LEASE_ERROR, - threshold_reason="watchdog artifact finalization failed", - error=f"{exc.component}: {exc}", - ) - try: - record = audit.emit("final", **fields) - except ArtifactError as nested_exc: - audit.disable_component(nested_exc.component) - record = audit.emit("final", **fields) - try: - audit.finalize(record) - except ArtifactError as nested_exc: - audit.disable_component(nested_exc.component) - exit_code = EXIT_LEASE_ERROR + record_artifact_error(exc) + emit_final_record() audit.mark_final(exit_code) return exit_code finally: @@ -1843,6 +1865,9 @@ def _graceful_cleanup( child_returncode, process_group_status, error, + preserve_primary_on_artifact_error=( + guardian_control_error is not None + ), ) diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index d1cfb7f3ef6a..8bb209fcbfb4 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -570,6 +570,32 @@ def test_parent_signal_allows_exit_after_pulse_deadline(self) -> None: def test_guardian_control_failure_still_kills_and_reaps_group( self, ) -> None: + class FailingFinalAudit: + def __init__(self, stream: Any): + self.stream = stream + self.write_count = 0 + + def write(self, value: str) -> int: + self.write_count += 1 + if self.write_count == 3: + raise OSError("final audit write failed") + return self.stream.write(value) + + def flush(self) -> None: + self.stream.flush() + + def fileno(self) -> int: + return self.stream.fileno() + + def close(self) -> None: + self.stream.close() + + class FailingFinalLease: + def finalize(self, record: dict[str, Any]) -> None: + raise watchdog.ArtifactError( + "lease", "final lease write failed" + ) + child_code = ( "import os,signal,sys,time;" "signal.signal(signal.SIGTERM,signal.SIG_IGN);" @@ -582,109 +608,142 @@ def test_guardian_control_failure_still_kills_and_reaps_group( " time.sleep(30)\n" ) for mode in ("closed", "blocked"): - with self.subTest(mode=mode): - with tempfile.TemporaryDirectory() as temp_dir: - pid_path = Path(temp_dir) / "pids" - process = subprocess.Popen( - [ - sys.executable, - "-c", - child_code, - str(pid_path), - ], - start_new_session=True, - ) - read_fd, write_fd = os.pipe() - os.set_blocking(write_fd, False) - if mode == "closed": - os.close(write_fd) - write_fd = -1 - else: - try: - while True: - os.write(write_fd, b"x" * 65536) - except BlockingIOError: - pass - guardian = watchdog.GuardianProcess( - process, - process.pid, - write_fd, + for artifact_failure in ("audit", "lease"): + with self.subTest( + mode=mode, + artifact_failure=artifact_failure, + ): + self._assert_guardian_control_failure_cleanup( + mode, + artifact_failure, + child_code, + FailingFinalAudit, + FailingFinalLease, ) - stream = io.StringIO() - audit = watchdog.AuditLogger(stream) - child_pid = None - grandchild_pid = None + + def _assert_guardian_control_failure_cleanup( + self, + mode: str, + artifact_failure: str, + child_code: str, + failing_final_audit: type, + failing_final_lease: type, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + pid_path = Path(temp_dir) / "pids" + process = subprocess.Popen( + [ + sys.executable, + "-c", + child_code, + str(pid_path), + ], + start_new_session=True, + ) + read_fd, write_fd = os.pipe() + os.set_blocking(write_fd, False) + if mode == "closed": + os.close(write_fd) + write_fd = -1 + else: + try: + while True: + os.write(write_fd, b"x" * 65536) + except BlockingIOError: + pass + guardian = watchdog.GuardianProcess( + process, + process.pid, + write_fd, + ) + stream = io.StringIO() + audit = watchdog.AuditLogger(stream) + if artifact_failure == "audit": + persistent_path = Path(temp_dir) / "persistent.jsonl" + persistent_stream = persistent_path.open( + "w", encoding="utf-8" + ) + audit.persistent_stream = failing_final_audit( + persistent_stream + ) + else: + audit.lease_manager = failing_final_lease() + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if time.monotonic() >= deadline: + self.fail("child process group did not start") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + result = watchdog._graceful_cleanup( + audit, + guardian, + snapshot(50), + 50, + "parent_signal", + 128 + signal.SIGTERM, + "wrapper received SIGTERM", + signal.SIGTERM, + 0.4, + watchdog._signal_process_group, + watchdog._process_group_alive, + time.monotonic, + time.sleep, + ) + finally: + if process.poll() is None: try: - deadline = time.monotonic() + 5 - while not pid_path.exists(): - if time.monotonic() >= deadline: - self.fail( - "child process group did not start" - ) - time.sleep(0.01) - child_pid, grandchild_pid = ( - int(value) - for value in pid_path.read_text( - encoding="utf-8" - ).split() - ) - result = watchdog._graceful_cleanup( - audit, - guardian, - snapshot(50), - 50, - "parent_signal", - 128 + signal.SIGTERM, - "wrapper received SIGTERM", - signal.SIGTERM, - 0.4, - watchdog._signal_process_group, - watchdog._process_group_alive, - time.monotonic, - time.sleep, - ) - finally: - if process.poll() is None: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=5) - guardian.close() - os.close(read_fd) + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + guardian.close() + os.close(read_fd) - records = [ - json.loads(line) - for line in stream.getvalue().splitlines() - ] - signals = [ - record["signal"] - for record in records - if record["event"] == "process_group_signal" - ] - self.assertEqual( - result, watchdog.EXIT_SIGNAL_ERROR - ) - self.assertEqual(signals, ["SIGTERM", "SIGKILL"]) - self.assertEqual( - records[-1]["classification"], "signal_error" - ) - self.assertEqual( - records[-1]["child_returncode"], - -signal.SIGKILL, - ) - assert child_pid is not None - assert grandchild_pid is not None - for process_id in (child_pid, grandchild_pid): - deadline = time.monotonic() + 2 - while ( - self._process_is_running(process_id) - and time.monotonic() < deadline - ): - time.sleep(0.01) - self.assertFalse( - self._process_is_running(process_id) - ) + records = [ + json.loads(line) + for line in stream.getvalue().splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual(result, watchdog.EXIT_SIGNAL_ERROR) + self.assertEqual(signals, ["SIGTERM", "SIGKILL"]) + self.assertEqual( + records[-1]["classification"], "signal_error" + ) + self.assertEqual(records[-1]["exit_code"], 7) + self.assertEqual( + records[-1]["threshold_reason"], + "guardian control failed during graceful cleanup", + ) + self.assertEqual( + records[-1]["secondary_errors"][0]["component"], + artifact_failure, + ) + self.assertEqual( + records[-1]["child_returncode"], + -signal.SIGKILL, + ) + assert child_pid is not None + assert grandchild_pid is not None + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) @unittest.skipUnless( sys.platform.startswith("linux"), From 778db6f50eae04e6c232c69b9575bdbd0747962b Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sat, 12 Sep 2026 23:56:55 -0700 Subject: [PATCH 18/56] scripts : retain watchdog artifact evidence Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/strix_memory_watchdog.py | 16 +++++++++++++ tests/test_strix_memory_watchdog.py | 37 +++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py index 75f3e8852e27..a06c85de3f96 100755 --- a/scripts/strix_memory_watchdog.py +++ b/scripts/strix_memory_watchdog.py @@ -1492,6 +1492,7 @@ def _emit_final( process_group_status: str = "not_created", error: str | None = None, preserve_primary_on_artifact_error: bool = False, + secondary_errors: Sequence[dict[str, str]] | None = None, ) -> int: fields = _state_fields( snapshot, @@ -1504,6 +1505,8 @@ def _emit_final( fields.update(classification=classification, exit_code=exit_code) if error: fields["error"] = error + if secondary_errors: + fields["secondary_errors"] = list(secondary_errors) def record_artifact_error(exc: ArtifactError) -> None: nonlocal exit_code @@ -1868,6 +1871,19 @@ def _graceful_cleanup( preserve_primary_on_artifact_error=( guardian_control_error is not None ), + secondary_errors=( + [ + { + "component": artifact_error.component, + "detail": str(artifact_error), + } + ] + if ( + guardian_control_error is not None + and artifact_error is not None + ) + else None + ), ) diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py index 8bb209fcbfb4..f9cba175a8d1 100644 --- a/tests/test_strix_memory_watchdog.py +++ b/tests/test_strix_memory_watchdog.py @@ -571,14 +571,15 @@ def test_guardian_control_failure_still_kills_and_reaps_group( self, ) -> None: class FailingFinalAudit: - def __init__(self, stream: Any): + def __init__(self, stream: Any, fail_at: int): self.stream = stream self.write_count = 0 + self.fail_at = fail_at def write(self, value: str) -> int: self.write_count += 1 - if self.write_count == 3: - raise OSError("final audit write failed") + if self.write_count == self.fail_at: + raise OSError("audit write failed") return self.stream.write(value) def flush(self) -> None: @@ -608,7 +609,12 @@ def finalize(self, record: dict[str, Any]) -> None: " time.sleep(30)\n" ) for mode in ("closed", "blocked"): - for artifact_failure in ("audit", "lease"): + for artifact_failure in ( + "term_audit", + "kill_audit", + "final_audit", + "lease", + ): with self.subTest( mode=mode, artifact_failure=artifact_failure, @@ -658,13 +664,18 @@ def _assert_guardian_control_failure_cleanup( ) stream = io.StringIO() audit = watchdog.AuditLogger(stream) - if artifact_failure == "audit": + if artifact_failure.endswith("_audit"): persistent_path = Path(temp_dir) / "persistent.jsonl" persistent_stream = persistent_path.open( "w", encoding="utf-8" ) audit.persistent_stream = failing_final_audit( - persistent_stream + persistent_stream, + { + "term_audit": 1, + "kill_audit": 2, + "final_audit": 3, + }[artifact_failure], ) else: audit.lease_manager = failing_final_lease() @@ -728,7 +739,19 @@ def _assert_guardian_control_failure_cleanup( ) self.assertEqual( records[-1]["secondary_errors"][0]["component"], - artifact_failure, + ( + "audit" + if artifact_failure.endswith("_audit") + else "lease" + ), + ) + self.assertIn( + ( + "audit write failed" + if artifact_failure.endswith("_audit") + else "final lease write failed" + ), + records[-1]["secondary_errors"][0]["detail"], ) self.assertEqual( records[-1]["child_returncode"], From 6d4acd62e4ecbb1e714134f1088687b454425ba0 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 00:04:06 -0700 Subject: [PATCH 19/56] deepseek41 : harden correctness trace admission Bind the independently approved watchdog v2 contract, enforce the admitted Strix runtime and audit evidence, and keep ds4 oracle execution fail closed until an exporter is pinned. Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 659 +++++++++++++++++- tools/deepseek-v41-trace/CMakeLists.txt | 1 + tools/deepseek-v41-trace/README.md | 211 +++++- tools/deepseek-v41-trace/llama-trace.cpp | 236 ++++++- tools/deepseek-v41-trace/preflight.py | 600 ++++++++++++++-- tools/deepseek-v41-trace/run_ds4.py | 25 +- tools/deepseek-v41-trace/run_llama.py | 66 +- tools/deepseek-v41-trace/run_matrix.py | 120 +++- tools/deepseek-v41-trace/trace_format.py | 297 +++++++- .../deepseek-v41-trace/verify_ds4_anchors.py | 112 +++ 10 files changed, 2166 insertions(+), 161 deletions(-) create mode 100644 tools/deepseek-v41-trace/verify_ds4_anchors.py diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 55339bb8c179..be301e1672d7 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -19,36 +19,98 @@ import run_llama import run_ds4 import preflight +import verify_ds4_anchors +trace.APPROVED_WATCHDOGS[trace.WATCHDOG_SCRIPT_SHA256] = trace.WATCHDOG_REVISION + +WATCHDOG_EVENTS = [ + { + "timestamp": "1970-01-01T00:00:01.000Z", + "event": "preflight", + "soft_bytes": trace.SOFT_MEMORY_LIMIT, + "emergency_bytes": trace.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": trace.STRICT_MEMORY_LIMIT, + "swap_entries": 0, + }, + { + "timestamp": "1970-01-01T00:00:01.000Z", + "event": "child_started", + "child_pid": 456, + "process_group_id": 455, + "command": ["python3", "run_matrix.py"], + }, +] +WATCHDOG_JSONL = "".join( + json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + for event in WATCHDOG_EVENTS +).encode("ascii") +WATCHDOG_JSONL_SHA256 = trace.sha256_bytes(WATCHDOG_JSONL) AUDIT_RECORDS = { "memory": { "created_unix": 1, "kind": "memory", + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, "data": {"mem_total_bytes": 128, "mem_available_bytes": 64, "mem_used_bytes": 64}, }, "swap": { "created_unix": 1, "kind": "swap", + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, "data": {"enabled": False, "entries": []}, }, "watchdog": { "created_unix": 1, "kind": "watchdog", + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, "data": { - "pid": 123, - "start_time_ticks": 456, - "command_sha256": "7" * 64, + "format": trace.WATCHDOG_LEASE_FORMAT, + "version": trace.WATCHDOG_VERSION, + "lease_id": "1" * 32, + "lease_path": "/run/user/123/watchdog.lease", + "watchdog_pid": 123, + "watchdog_start_time_ticks": 456, + "watchdog_command": "python3 /repo/scripts/strix_memory_watchdog.py", + "watchdog_command_sha256": "7" * 64, + "watchdog_executable_path": "/usr/bin/python3", + "watchdog_script_path": "/repo/scripts/strix_memory_watchdog.py", + "watchdog_script_sha256": trace.WATCHDOG_SCRIPT_SHA256, + "watchdog_revision": trace.WATCHDOG_REVISION, + "soft_bytes": trace.SOFT_MEMORY_LIMIT, + "emergency_bytes": trace.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": trace.STRICT_MEMORY_LIMIT, + "grace_seconds": 30.0, + "sample_interval_seconds": 1.0, + "procfs_root": "/proc", + "guardian_pid": 455, + "child_pid": 456, + "child_process_group_id": 455, + "command": ["python3", "run_matrix.py"], + "child_command_sha256": trace.sha256_bytes(b'["python3","run_matrix.py"]'), "heartbeat_path": "/run/user/123/watchdog.heartbeat", "heartbeat_unix": 1, - "max_heartbeat_age_seconds": 30, + "max_heartbeat_age_seconds": 5.0, + "audit_live_path": "/run/user/123/watchdog.jsonl", + "audit_device": 1, + "audit_inode": 2, + "audit_uid": 1000, + "audit_mode": 0o600, + "audit_fd": 3, + "audit": { + "path": "", + "sha256": WATCHDOG_JSONL_SHA256, + "event_count": len(WATCHDOG_EVENTS), + }, }, }, } -def audit_bytes(kind: str) -> bytes: - return (json.dumps(AUDIT_RECORDS[kind], sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") +def audit_bytes(kind: str, phase: str) -> bytes: + record = json.loads(json.dumps(AUDIT_RECORDS[kind])) + if kind == "watchdog": + record["data"]["audit"]["path"] = f"audits/{phase}/{WATCHDOG_JSONL_SHA256}.jsonl" + return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") def provenance_bytes(prompt: bytes = b"abc") -> bytes: @@ -88,13 +150,16 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "config": { "context": 3, "decode_steps": 1, - "batch": 512, - "ubatch": 128, + "batch": trace.ADMITTED_BATCH, + "ubatch": trace.ADMITTED_UBATCH, "kv_type_k": "f16", "kv_type_v": "f16", "flash_attention": True, - "expert_cache_slots": 8, - "expert_cache_bytes": 4096, + "expert_cache_slots": trace.REQUIRED_EXPERT_SLOTS, + "expert_cache_bytes": trace.REQUIRED_EXPERT_CACHE_BYTES, + "device": "ROCm0", + "gpu_layers": 99, + "load_mode": 0, "deepseek41": { "layer_count": 40, "vocab_size": 129280, @@ -106,6 +171,8 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "candidate_topk_blocks": 2048, "candidate_block_size": 8, "index_top_k": 512, + "raw_attention_layers": list(trace.RAW_ATTENTION_LAYERS), + "raw_attention_width": trace.RAW_ATTENTION_WIDTH, "candidate_propagation_layers": [24, 28, 32, 36], }, }, @@ -131,8 +198,8 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "audits": { phase: { kind: { - "path": f"audits/{phase}/{trace.sha256_bytes(audit_bytes(kind))}.json", - "sha256": trace.sha256_bytes(audit_bytes(kind)), + "path": f"audits/{phase}/{trace.sha256_bytes(audit_bytes(kind, phase))}.json", + "sha256": trace.sha256_bytes(audit_bytes(kind, phase)), "created_unix": 1, } for kind in ("memory", "swap", "watchdog") @@ -148,6 +215,8 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "diff_sha256": "c" * 64, "executable_sha256": "3" * 64, } + else: + result["config"]["prefill_chunk"] = trace.ADMITTED_UBATCH return result @@ -156,8 +225,9 @@ def add_required_events(writer: object, logits: bytes | None = None, prompt: byt audit_root = writer.root / "audits" / phase audit_root.mkdir(parents=True, exist_ok=True) for kind in ("memory", "swap", "watchdog"): - data = audit_bytes(kind) + data = audit_bytes(kind, phase) (audit_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) + (audit_root / f"{WATCHDOG_JSONL_SHA256}.jsonl").write_bytes(WATCHDOG_JSONL) provenance_root = writer.root / "provenance" provenance_root.mkdir(exist_ok=True) data = provenance_bytes(prompt) @@ -218,6 +288,21 @@ def add_required_events(writer: object, logits: bytes | None = None, prompt: byt shape=[6, 2], data=struct.pack(" None: + events_path = root / trace.EVENTS_NAME + events = [json.loads(line) for line in events_path.read_text(encoding="ascii").splitlines()] + event = next( + item for item in events + if item["component"] == component and item["phase"] == phase and item["layer"] == layer + ) + digest = trace.sha256_bytes(data) + blob = root / trace.BLOBS_DIR / f"{digest}.bin" + blob.write_bytes(data) + event.update({ + "shape": shape, + "byte_count": len(data), + "sha256": digest, + "blob": f"{trace.BLOBS_DIR}/{digest}.bin", + }) + events_path.write_text( + "".join(json.dumps(item, sort_keys=True, separators=(",", ":")) + "\n" for item in events), + encoding="ascii", + ) + + class TraceFormatTests(unittest.TestCase): def test_serialization_preserves_float_bits_and_hashes(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -437,15 +566,288 @@ def test_llama_runner_preserves_binary_prompt_bytes(self) -> None: context=32768, decode_steps=8, batch=2048, - ubatch=512, + ubatch=trace.ADMITTED_UBATCH, + device="ROCm0", gpu_layers=99, - expert_cache_slots=8, - expert_cache_mib=4096, + expert_cache_slots=trace.REQUIRED_EXPERT_SLOTS, + expert_cache_mib=trace.REQUIRED_EXPERT_CACHE_MIB, ) command = run_llama.build_command(args, exporter, output) self.assertEqual(command[command.index("-bf") + 1], str(prompt.resolve())) + self.assertEqual(command[command.index("--device") + 1], "ROCm0") + self.assertEqual(command[command.index("--load-mode") + 1], "none") self.assertNotIn("-f", command) + def test_rejects_unadmitted_expert_cache_configuration(self) -> None: + invalid = Namespace( + batch=trace.ADMITTED_BATCH, + ubatch=512, + device="ROCm0", + gpu_layers=99, + expert_cache_slots=8, + expert_cache_mib=4096, + ) + with self.assertRaisesRegex(preflight.PreflightError, "admitted ubatch 32"): + run_llama.validate_runtime_config(invalid) + + invalid.ubatch = trace.ADMITTED_UBATCH + with self.assertRaisesRegex(preflight.PreflightError, "192 expert cache slots"): + run_llama.validate_runtime_config(invalid) + + invalid.expert_cache_slots = trace.REQUIRED_EXPERT_SLOTS + with self.assertRaisesRegex(preflight.PreflightError, "76441190400 expert cache bytes"): + run_llama.validate_runtime_config(invalid) + + valid = Namespace( + batch=trace.ADMITTED_BATCH, + ubatch=trace.ADMITTED_UBATCH, + device="ROCm0", + gpu_layers=99, + expert_cache_slots=trace.REQUIRED_EXPERT_SLOTS, + expert_cache_mib=trace.REQUIRED_EXPERT_CACHE_MIB, + ) + run_llama.validate_runtime_config(valid) + self.assertEqual(trace.REQUIRED_EXPERT_CACHE_BYTES, 76_441_190_400) + self.assertEqual(trace.REQUIRED_EXPERT_CACHE_MIB, 72_900) + + def test_watchdog_lease_rejects_arbitrary_heartbeat_process(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + repo = root / "repo" + script = repo / "scripts" / "strix_memory_watchdog.py" + script.parent.mkdir(parents=True) + script.write_text("#!/usr/bin/env python3\n", encoding="ascii") + + procfs = root / "proc" + watchdog_pid = 123 + child_pid = 456 + current_pid = 789 + for pid in (watchdog_pid, child_pid, current_pid): + (procfs / str(pid)).mkdir(parents=True) + + def stat(pid: int, parent: int, start: int) -> str: + fields = ["S", str(parent), *(["0"] * 17), str(start)] + return f"{pid} (test) " + " ".join(fields) + "\n" + + (procfs / str(watchdog_pid) / "stat").write_text( + stat(watchdog_pid, 1, 1000), encoding="ascii") + (procfs / str(child_pid) / "stat").write_text( + stat(child_pid, watchdog_pid, 2000), encoding="ascii") + (procfs / str(current_pid) / "stat").write_text( + stat(current_pid, child_pid, 3000), encoding="ascii") + + watchdog_command = ( + b"python3\0" + str(script.resolve()).encode("ascii") + + b"\0--soft-gib\0" + b"116\0--emergency-gib\0" + b"118\0" + ) + child_command = b"python3\0run_matrix.py\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(watchdog_command) + (procfs / str(child_pid) / "cmdline").write_bytes(child_command) + (procfs / str(watchdog_pid) / "cwd").symlink_to(repo) + + heartbeat = root / "heartbeat.json" + audit = root / "watchdog.jsonl" + lease = root / "lease.json" + lease_id = "1" * 32 + heartbeat.write_text(json.dumps({ + "format": preflight.WATCHDOG_HEARTBEAT_FORMAT, + "version": preflight.WATCHDOG_VERSION, + "lease_id": lease_id, + "sequence": 1, + "state": "active", + "updated_at": "1970-01-01T00:01:40.000Z", + "updated_monotonic_ns": 1, + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "child_pid": child_pid, + "child_process_group_id": child_pid, + "sample": {}, + }), encoding="ascii") + child_argv = ["python3", "run_matrix.py"] + audit.write_text( + json.dumps({ + "event": "preflight", + "soft_bytes": preflight.SOFT_MEMORY_LIMIT, + "emergency_bytes": preflight.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": preflight.STRICT_MEMORY_LIMIT, + "swap_entries": 0, + }) + "\n" + + json.dumps({ + "event": "child_started", + "child_pid": child_pid, + "process_group_id": child_pid, + "command": child_argv, + }) + "\n", + encoding="ascii", + ) + lease_record = { + "format": preflight.WATCHDOG_LEASE_FORMAT, + "version": preflight.WATCHDOG_VERSION, + "lease_id": lease_id, + "state": "active", + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "watchdog_command_sha256": preflight.sha256_bytes(watchdog_command), + "watchdog_script_path": str(script.resolve()), + "watchdog_script_sha256": preflight.sha256_bytes(script.read_bytes()), + "soft_bytes": preflight.SOFT_MEMORY_LIMIT, + "emergency_bytes": preflight.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": preflight.STRICT_MEMORY_LIMIT, + "procfs_root": "/proc", + "child_pid": child_pid, + "child_process_group_id": child_pid, + "command": child_argv, + "child_command_sha256": preflight.sha256_bytes(child_command), + "heartbeat_path": str(heartbeat), + "max_heartbeat_age_seconds": 5.0, + "audit_path": str(audit), + } + lease_record["child_command_sha256"] = preflight.sha256_bytes( + json.dumps(child_argv, ensure_ascii=True, separators=(",", ":")).encode("utf-8")) + lease.write_text(json.dumps(lease_record), encoding="ascii") + environment = { + preflight.WATCHDOG_LEASE_ENV: str(lease), + preflight.WATCHDOG_HEARTBEAT_ENV: str(heartbeat), + preflight.WATCHDOG_AUDIT_ENV: str(audit), + preflight.WATCHDOG_MAX_AGE_ENV: "5.0", + } + result = preflight.watchdog_audit( + repo, + environment=environment, + procfs_root=procfs, + current_pid=current_pid, + current_pgid=child_pid, + getpgid=lambda pid: child_pid, + now=100, + monotonic_ns=lambda: 1_000_000_001, + timeout_seconds=0, + ) + self.assertEqual(result["child_process_group_id"], child_pid) + + arbitrary = root / "arbitrary-heartbeat.py" + arbitrary.write_text("#!/usr/bin/env python3\n", encoding="ascii") + arbitrary_command = b"python3\0" + str(arbitrary.resolve()).encode("ascii") + b"\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(arbitrary_command) + lease_record["watchdog_command_sha256"] = preflight.sha256_bytes(arbitrary_command) + lease.write_text(json.dumps(lease_record), encoding="ascii") + with self.assertRaisesRegex(preflight.PreflightError, "candidate repository script"): + preflight.watchdog_audit( + repo, + environment=environment, + procfs_root=procfs, + current_pid=current_pid, + current_pgid=child_pid, + getpgid=lambda pid: child_pid, + now=100, + monotonic_ns=lambda: 1_000_000_001, + timeout_seconds=0, + ) + + def test_canonical_watchdog_validation_is_pinned_and_delegated(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + repo = root / "repo" + script = repo / "scripts" / "strix_memory_watchdog.py" + script.parent.mkdir(parents=True) + script.write_text("# fixture\n", encoding="ascii") + lease = root / "lease.json" + heartbeat = root / "heartbeat.json" + audit = root / "audit.jsonl" + lease.write_text("{}\n", encoding="ascii") + heartbeat.write_text( + json.dumps({"updated_at": "1970-01-01T00:00:01.000Z"}) + "\n", + encoding="ascii", + ) + audit.write_bytes(WATCHDOG_JSONL) + environment = { + preflight.WATCHDOG_LEASE_ENV: str(lease), + preflight.WATCHDOG_HEARTBEAT_ENV: str(heartbeat), + preflight.WATCHDOG_AUDIT_ENV: str(audit), + preflight.WATCHDOG_MAX_AGE_ENV: "5", + } + + class FakeLeaseError(RuntimeError): + pass + + class FakeWatchdog: + LeaseValidationError = FakeLeaseError + calls = [] + + @classmethod + def validate_active_lease(cls, lease_path: Path, **kwargs: object) -> dict[str, object]: + cls.calls.append((lease_path, kwargs)) + return { + **AUDIT_RECORDS["watchdog"]["data"], + "audit_path": str(audit), + "heartbeat_path": str(heartbeat), + "child_pid": 456, + } + + @staticmethod + def start_process_group_lease_guard(*args: object, **kwargs: object) -> None: + raise AssertionError("descendant validation must not start the direct-child guard") + + original_sha256 = preflight.WATCHDOG_SCRIPT_SHA256 + original_approved = dict(preflight.APPROVED_WATCHDOGS) + try: + preflight.WATCHDOG_SCRIPT_SHA256 = preflight.sha256_bytes(script.read_bytes()) + preflight.APPROVED_WATCHDOGS[preflight.WATCHDOG_SCRIPT_SHA256] = ( + preflight.WATCHDOG_REVISION) + result = preflight.watchdog_audit( + repo, + environment=environment, + procfs_root=Path("/proc"), + current_pid=789, + watchdog_module=FakeWatchdog, + monotonic=lambda: 2.0, + sleeper=lambda _: None, + ) + finally: + preflight.APPROVED_WATCHDOGS.clear() + preflight.APPROVED_WATCHDOGS.update(original_approved) + preflight.WATCHDOG_SCRIPT_SHA256 = original_sha256 + self.assertEqual(result["watchdog_revision"], preflight.WATCHDOG_REVISION) + self.assertEqual(len(FakeWatchdog.calls), 1) + kwargs = FakeWatchdog.calls[0][1] + self.assertEqual(kwargs["expected_soft_bytes"], trace.SOFT_MEMORY_LIMIT) + self.assertEqual(kwargs["expected_emergency_bytes"], trace.WATCHDOG_EMERGENCY_LIMIT) + self.assertEqual(kwargs["expected_procfs_root"], Path("/proc")) + + def test_approved_watchdog_is_exact(self) -> None: + expected = { + "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f": + "778db6f50eae04e6c232c69b9575bdbd0747962b", + } + self.assertEqual(preflight.APPROVED_WATCHDOGS, expected) + self.assertEqual(trace.APPROVED_WATCHDOGS, expected) + self.assertEqual(preflight.WATCHDOG_VERSION, 2) + self.assertEqual(trace.WATCHDOG_VERSION, 2) + + def test_workload_scan_ignores_guarded_process_ancestry(self) -> None: + with tempfile.TemporaryDirectory() as temp: + procfs = Path(temp) + + def add_process(pid: int, parent: int, command: bytes) -> None: + process = procfs / str(pid) + process.mkdir() + process.joinpath("stat").write_text( + f"{pid} (test) S {parent} " + " ".join(["0"] * 18) + "\n", + encoding="ascii", + ) + process.joinpath("cmdline").write_bytes(command) + + add_process(90, 1, b"python3\0scripts/strix_memory_watchdog.py\0DeepSeek-V4.1\0") + add_process(100, 90, b"python3\0run_matrix.py\0--ds4-checkout\0/home/papa/src/ds4-v41\0") + add_process(200, 1, b"/tmp/ds4-v41-worker\0") + self.assertEqual( + preflight.matching_workloads( + ["ds4-v41", "DeepSeek-V4.1"], + procfs_root=procfs, + current_pid=100, + ), + [{"pid": 200, "command": "/tmp/ds4-v41-worker"}], + ) + def test_rejects_cache_slot_id_space(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" @@ -559,6 +961,93 @@ def test_rejects_empty_variable_width_components(self) -> None: ) writer.events.close() + def test_rejects_malformed_raw_attention_sources(self) -> None: + cases = ( + ( + "width", + 0, + [trace.RAW_ATTENTION_WIDTH - 1, 2], + [trace.RAW_ATTENTION_WIDTH + 2] * ((trace.RAW_ATTENTION_WIDTH - 1) * 2), + "raw attn.source shape", + ), + ( + "future ubatch row", + 0, + [trace.RAW_ATTENTION_WIDTH, 2], + [trace.RAW_ATTENTION_WIDTH + 1] + + [trace.RAW_ATTENTION_WIDTH + 2] * (trace.RAW_ATTENTION_WIDTH * 2 - 1), + "invalid row", + ), + ( + "layer mismatch", + 1, + [trace.RAW_ATTENTION_WIDTH, 2], + [0] + [trace.RAW_ATTENTION_WIDTH + 2] * (trace.RAW_ATTENTION_WIDTH * 2 - 1), + "differs between layers 0 and 1", + ), + ) + for name, layer, shape, values, message in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + replace_event_blob( + root, + component="attn.source", + phase="prefill", + layer=layer, + shape=shape, + data=struct.pack("<" + "i" * len(values), *values), + ) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_compares_raw_attention_sources_in_every_prefill_chunk(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + events = [ + json.loads(line) + for line in (root / trace.EVENTS_NAME).read_text(encoding="ascii").splitlines() + ] + split_events = [] + for event in events: + if event["component"] != "attn.source" or event["phase"] != "prefill" or ( + event["layer"] not in trace.RAW_ATTENTION_LAYERS): + split_events.append(event) + continue + for token_start in (0, 1): + values = [0] * trace.RAW_ATTENTION_WIDTH + if event["layer"] == 0 and token_start == 0: + values[0] = 1 + data = struct.pack("<" + "i" * len(values), *values) + digest = trace.sha256_bytes(data) + (root / trace.BLOBS_DIR / f"{digest}.bin").write_bytes(data) + split = dict(event) + split.update({ + "token_start": token_start, + "token_count": 1, + "shape": [trace.RAW_ATTENTION_WIDTH, 1], + "byte_count": len(data), + "sha256": digest, + "blob": f"{trace.BLOBS_DIR}/{digest}.bin", + }) + split_events.append(split) + (root / trace.EVENTS_NAME).write_text( + "".join(json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + for event in split_events), + encoding="ascii", + ) + manifest_record = json.loads((root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + manifest_record["event_count"] = len(split_events) + (root / trace.MANIFEST_NAME).write_text( + json.dumps(manifest_record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + with self.assertRaisesRegex(trace.TraceError, "differs between layers 0 and 1"): + trace.TraceBundle(root) + def test_rejects_zero_dimensions_globally(self) -> None: with tempfile.TemporaryDirectory() as temp: writer = trace.TraceBundleWriter(Path(temp) / "trace", manifest()) @@ -586,6 +1075,106 @@ def test_rejects_dirty_ds4_checkout(self) -> None: finally: run_ds4.git_output = original + def test_verifies_pinned_ds4_anchor_hashes(self) -> None: + with tempfile.TemporaryDirectory() as temp: + checkout = Path(temp) + anchor = checkout / "tests" / "fixture.vec" + anchor.parent.mkdir(parents=True) + anchor.write_bytes(b"fixture") + expected = trace.sha256_bytes(b"fixture") + original = verify_ds4_anchors.git_output + try: + verify_ds4_anchors.git_output = lambda checkout, *args: ( + trace.DS4_REVISION if args == ("rev-parse", "HEAD") else "") + result = verify_ds4_anchors.verify( + checkout, + anchors={"tests/fixture.vec": expected}, + ) + self.assertEqual(result["status"], "ANCHORS VERIFIED") + anchor.write_bytes(b"changed") + with self.assertRaisesRegex(verify_ds4_anchors.AnchorError, "SHA-256 mismatch"): + verify_ds4_anchors.verify( + checkout, + anchors={"tests/fixture.vec": expected}, + ) + finally: + verify_ds4_anchors.git_output = original + + def test_embeds_rewritten_watchdog_audit_content(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "source" + source.mkdir() + audits = {} + for kind in ("memory", "swap", "watchdog"): + record = json.loads(json.dumps(AUDIT_RECORDS[kind])) + if kind == "watchdog": + jsonl = source / "watchdog-events.jsonl" + jsonl.write_bytes(WATCHDOG_JSONL) + record["data"].pop("audit") + record["data"]["audit_path"] = str(jsonl) + record["data"]["audit_sha256"] = WATCHDOG_JSONL_SHA256 + record["data"]["audit_event_count"] = len(WATCHDOG_EVENTS) + path = source / f"{kind}.json" + path.write_text( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + audits[kind] = str(path) + references = preflight.embed_audits(root / "trace", "pre", audits) + embedded = json.loads( + (root / "trace" / references["watchdog"]["path"]).read_text(encoding="ascii")) + self.assertIn("audit", embedded["data"]) + self.assertNotIn("audit_path", embedded["data"]) + self.assertEqual( + trace.sha256_bytes( + (root / "trace" / references["watchdog"]["path"]).read_bytes()), + references["watchdog"]["sha256"], + ) + + def test_rejects_unapproved_ds4_exporter(self) -> None: + with self.assertRaisesRegex(preflight.PreflightError, "not approved"): + run_ds4.verify_exporter_approval("a" * 64) + + def test_rejects_preflight_audit_mutation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + audits = {} + for kind in ("memory", "swap", "watchdog"): + record = json.loads(json.dumps(AUDIT_RECORDS[kind])) + if kind == "watchdog": + jsonl = root / "watchdog-events.jsonl" + jsonl.write_bytes(WATCHDOG_JSONL) + record["data"]["audit_path"] = str(jsonl) + record["data"]["audit_sha256"] = WATCHDOG_JSONL_SHA256 + record["data"]["audit_event_count"] = len(WATCHDOG_EVENTS) + path = root / f"{kind}.json" + path.write_text( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + audits[kind] = str(path) + digests = preflight.seal_audits(audits) + memory = Path(audits["memory"]) + memory.chmod(0o644) + memory.write_text("{}\n", encoding="ascii") + with self.assertRaisesRegex(preflight.PreflightError, "changed during runtime"): + preflight.verify_sealed_audits(audits, digests) + + def test_rejects_symlinked_bundle_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + bundle = root / "bundle" + with trace.TraceBundleWriter(bundle, manifest()) as writer: + add_required_events(writer) + provenance = next((bundle / "provenance").iterdir()) + outside = root / "outside.json" + outside.write_bytes(provenance.read_bytes()) + provenance.unlink() + provenance.symlink_to(outside) + with self.assertRaisesRegex(trace.TraceError, "must not use symlinks"): + trace.TraceBundle(bundle) + def test_rejects_weakened_coverage_contract(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" @@ -623,6 +1212,42 @@ def test_report_generation_passes_identical_bundles(self) -> None: self.assertEqual(result["events_compared"], 259) self.assertIsNone(result["first_divergence"]) + def test_local_bringup_reports_do_not_claim_cross_runtime_pass(self) -> None: + with tempfile.TemporaryDirectory() as temp: + first = Path(temp) / "first" + second = Path(temp) / "second" + with trace.TraceBundleWriter(first, manifest("llama.cpp")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(second, manifest("llama.cpp")) as writer: + add_required_events(writer) + result = trace.local_report( + trace.TraceBundle(first), + trace.TraceBundle(second), + "self-consistency", + ) + self.assertEqual(result["status"], "BRINGUP PASS") + self.assertNotEqual(result["status"], "TARGET PASS") + self.assertEqual(result["cross_runtime_status"], "INCOMPLETE") + + def test_local_base_regression_requires_attested_oracle_revision(self) -> None: + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / "base" + integrated = Path(temp) / "integrated" + base_manifest = manifest("llama.cpp") + base_manifest["revision"] = "b" * 40 + base_manifest["candidate"]["revision"] = "b" * 40 + with trace.TraceBundleWriter(base, base_manifest) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(integrated, manifest("llama.cpp")) as writer: + add_required_events(writer) + result = trace.local_report( + trace.TraceBundle(base), + trace.TraceBundle(integrated), + "base-regression", + ) + self.assertEqual(result["status"], "BRINGUP PASS") + self.assertEqual(result["cross_runtime_status"], "INCOMPLETE") + def test_manifest_mismatch_is_classified(self) -> None: with tempfile.TemporaryDirectory() as temp: left = Path(temp) / "left" diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index 3f311dd4410f..b54dea2d97ec 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -1,6 +1,7 @@ set(TARGET llama-deepseek-v41-trace) add_executable(${TARGET} llama-trace.cpp) target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src) target_compile_features(${TARGET} PRIVATE cxx_std_17) set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index ddadb578308a..2663e6b96164 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -12,7 +12,7 @@ Each trace is a directory: The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. -Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, attention-source IDs as nonempty rank-2 i32 with width at most 512 and `token_count` in the second dimension, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Original expert IDs must be within `0..383`. +Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, layer-0/1 raw attention-source rows as i32 `[128, token_count]`, compressed attention-source IDs as nonempty rank-2 i32 with width at most 512, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Raw attention rows use physical ring IDs `0..127`, visible current-ubatch IDs `128..128+token_index`, and unavailable sentinel `128+token_count`; layers 0 and 1 must be byte-identical for each execution step. Original expert IDs must be within `0..383`. Validate or compare bundles: @@ -38,35 +38,208 @@ Their SHA-256 values are fixed in `trace_format.py`; the matrix refuses modified Start at context 32768. `llama-deepseek-v41-prompt-builder` loads only the GGUF vocabulary, repeats each repository corpus deterministically, truncates the token sequence to `context - decode_steps`, detokenizes it, and requires exact token round-trip before saving the prompt. `run_matrix.py` creates each prompt once before either runtime executes and reuses its exact bytes for every chunk-boundary case. Run prefill plus at least eight greedy decode steps in one reused context. -## Strix execution gate +The initial correctness matrix uses the final admitted physical ubatch of 32. The worst-case routed expert union is `min(384, 6*32) = 192` experts per layer. The published GGUF metadata yields 398131200 bytes per cross-layer expert slot, so the required cache is exactly 76441190400 bytes, or 72900 MiB. The launchers reject other ubatch, slot, or cache-byte values. Host-memory admission still accounts the cache, staging, model, graph, state, outputs, current host use, and safety margin together before allocation. -`run_ds4.py` verifies that the pinned ds4 checkout has no tracked or untracked changes and refuses model execution when swap is enabled, the watchdog lease or heartbeat is missing/stale, another matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. +Run the oracle matrix only from the final integration commit that contains the canonical watchdog, memory admission, and this correctness harness. Supply its exact revision, its immutable merge-base, and a newly computed base-to-head binary diff SHA-256 to `run_matrix.py`; do not reuse the standalone correctness PR head or diff identity. -The watchdog lease is JSON, not a bare PID: +ROCm on `gfx1151` is the primary acceptance backend: -```json -{"pid":1234,"start_time_ticks":5678,"command_sha256":"","heartbeat_path":"/run/user/1000/dsv41-watchdog.heartbeat","max_heartbeat_age_seconds":30} +```sh +HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ + cmake -S . -B build-dsv41-trace-rocm \ + -DGGML_HIP=ON \ + -DGPU_TARGETS=gfx1151 \ + -DGGML_NATIVE=ON \ + -DCMAKE_BUILD_TYPE=Release +cmake --build build-dsv41-trace-rocm --config Release -j "$(nproc)" --target \ + llama-deepseek-v41-trace \ + llama-deepseek-v41-prompt-builder \ + test-backend-ops \ + test-deepseek41-schema \ + test-deepseek41-engram \ + test-deepseek41-expert \ + test-deepseek41-memory \ + test-deepseek41-runtime + +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o MUL_MAT_ID +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o MUL_MAT +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o GET_ROWS +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o SET_ROWS +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o CPY ``` -The watchdog must update the heartbeat file with the current Unix timestamp at least every 30 seconds. The wrappers verify the PID, Linux process start time, exact command bytes, and heartbeat before and after execution. The llama exporter repeats the same identity and heartbeat check before finalizing its trace. +Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly by default. + +Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the complete matrix process group. The wrappers fail closed if this variable is absent or different, and every embedded memory, swap, and watchdog audit records it. Keep the same inherited value for ds4 and llama.cpp. + +## Strix execution gate + +`run_ds4.py` verifies that the pinned ds4 checkout has no tracked or untracked changes and refuses model execution when swap is enabled, the canonical watchdog lease, heartbeat, or JSONL audit is missing or stale, another unrelated matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. + +The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd0747962b`, with `scripts/strix_memory_watchdog.py` SHA-256 `d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f`. Both Python validators and the native exporter reject every other revision or script hash. + +The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. + +Use one empty directory on verified non-rotational NVMe for every input and output. These metadata commands do not execute the model: ```sh -python3 tools/deepseek-v41-trace/run_ds4.py \ - --model /mnt/models/deepseek-v41/DeepSeek-V4.1-Flash-Q2.gguf \ - --prompt /path/on/nvme/correctness-prose-32768.txt \ - --corpus-name correctness-prose.txt \ - --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ - --prompt-provenance /path/on/nvme/correctness-prose-32768.txt.provenance.json \ - --output /path/on/nvme/traces/ds4-prose-32768 \ - --watchdog-pid-file /run/user/$(id -u)/dsv41-watchdog.pid \ - --exporter /path/to/pinned-ds4-trace-exporter \ - --exporter-sha256 +MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf +REPO=/home/papa/src/strix-llama-integration +RUN_ROOT=/home/papa/dsv41-correctness +TMPDIR=/home/papa/tmp/dsv41 +export TMPDIR + +test "$(stat -c %s "$MODEL")" = 365713686528 +test "$(realpath "$MODEL")" = /mnt/models/DeepSeek-V4.1-Flash-Q2.gguf +test "$(realpath -m "$REPO")" = /home/papa/src/strix-llama-integration +test "$(realpath -m "$RUN_ROOT")" = /home/papa/dsv41-correctness +test "$(realpath -m "$TMPDIR")" = /home/papa/tmp/dsv41 +mkdir -p "$RUN_ROOT" "$TMPDIR" +findmnt -no SOURCE,FSTYPE,TARGET -T "$MODEL" +findmnt -no SOURCE,FSTYPE,TARGET -T /home +lsblk -d -o NAME,ROTA,TYPE,SIZE,MODEL +df -B1 /mnt/models /home +sha256sum "$MODEL" +PYTHONPATH=gguf-py python3 -m gguf.scripts.gguf_dump "$MODEL" > "$RUN_ROOT/model-metadata.txt" +git -C /home/papa/src/ds4-v41 status --short +git -C /home/papa/src/ds4-v41 rev-parse HEAD +test "$(awk 'NR > 1 { count++ } END { print count + 0 }' /proc/swaps)" = 0 ``` +The expected model digest is `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`, the ds4 status output is empty, the ds4 revision is `bd66c402070042bf0a79ad6ece8242de4c93680c`, both selected block devices report `ROTA=0`, and `/proc/swaps` has zero entries. The observed planning snapshot had 76366495744 bytes free on `/mnt/models` and 679635001344 bytes free under `/home`; recheck before every run. Keep the unchanged 365713686528-byte GGUF in place on `/mnt/models`. Do not copy the model or place builds, logs, traces, audit files, or temporary files there. Put all of those under `/home`, and never use `/mnt/bigspace`. + The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision and its build SHA-256 matches the executed file. The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact candidate revision, full-graph base revision, expected base-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision or executable hash does not match that attestation. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision or executable hash does not match that attestation. + +`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. + +The external ds4 exporter is not present in the pinned `/home/papa/src/ds4-v41` checkout. It remains a blocker until a separately built executable is provided and attested. `run_ds4.py` currently has no approved exporter digest and fails closed before inference. After the exporter is implemented and reviewed, add its exact executable SHA-256 and pinned ds4 revision to `APPROVED_EXPORTERS` in `run_ds4.py`; a caller-provided digest alone is not sufficient oracle provenance. The exporter must accept the interface used by `run_ds4.py`: + +The unpublished `ds4gguf` documentation revision `e13893ffcb33e90c8852929303e188102df7a8f5` is provenance only. It is not an executable dependency, exporter approval, or fixture source. Executable tests and fixtures stay in this `strix-llama.cpp` stack. + +```text +--model PATH --prompt-file PATH --output PATH --context N --decode-steps N --prefill-chunk 32 +--memory-audit PATH --swap-audit PATH --watchdog-audit PATH +``` + +It must emit a complete valid `dsv41-trace` bundle, report ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`, and put its own executable SHA-256 in `manifest.json`. + +After the exporter exists, set the immutable identities and run the first matrix under the watchdog: + +```sh +REPO=/home/papa/src/strix-llama-integration +MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf +RUN_ROOT=/home/papa/dsv41-correctness +CASE_ROOT="$RUN_ROOT/c32768" +DS4_EXPORTER=/home/papa/bin/dsv41-trace-exporter +CANDIDATE_REV="$(git -C "$REPO" rev-parse HEAD)" +BASE_REV= +DIFF_SHA256="$(git -C "$REPO" diff --binary --no-ext-diff "$BASE_REV" "$CANDIDATE_REV" -- | sha256sum | awk '{print $1}')" +DS4_EXPORTER_SHA256="$(sha256sum "$DS4_EXPORTER" | awk '{print $1}')" + +mkdir -p "$CASE_ROOT/watchdog" +cd "$REPO" +HIP_LAUNCH_BLOCKING=1 python3 scripts/strix_memory_watchdog.py \ + --procfs-root /proc \ + --soft-gib 116 \ + --emergency-gib 118 \ + --grace-seconds 30 \ + --sample-interval-seconds 1 \ + --lease-path "$CASE_ROOT/watchdog/lease.json" \ + --heartbeat-path "$CASE_ROOT/watchdog/heartbeat.json" \ + --audit-path "$CASE_ROOT/watchdog/audit.jsonl" \ + --heartbeat-max-age-seconds 5 \ + -- \ + python3 tools/deepseek-v41-trace/run_matrix.py \ + --repo "$REPO" \ + --model "$MODEL" \ + --output "$CASE_ROOT/matrix" \ + --llama-runner "$REPO/tools/deepseek-v41-trace/run_llama.py" \ + --llama-exporter "$REPO/build-dsv41-trace-rocm/bin/llama-deepseek-v41-trace" \ + --llama-prompt-builder "$REPO/build-dsv41-trace-rocm/bin/llama-deepseek-v41-prompt-builder" \ + --candidate-revision "$CANDIDATE_REV" \ + --base-revision "$BASE_REV" \ + --candidate-diff-sha256 "$DIFF_SHA256" \ + --ds4-runner "$REPO/tools/deepseek-v41-trace/run_ds4.py" \ + --ds4-checkout /home/papa/src/ds4-v41 \ + --ds4-exporter "$DS4_EXPORTER" \ + --ds4-exporter-sha256 "$DS4_EXPORTER_SHA256" \ + --contexts 32768 \ + --ubatches 32 \ + --batch 2048 \ + --device ROCm0 \ + --expert-cache-slots 192 \ + --expert-cache-mib 72900 +``` + +Use a new empty output and watchdog directory for each later context. Repeat the same command after setting `CASE_ROOT` and changing `--contexts`: + +```sh +CASE_ROOT="$RUN_ROOT/c65536" # then use --contexts 65536 +CASE_ROOT="$RUN_ROOT/c98304" # then use --contexts 98304 +CASE_ROOT="$RUN_ROOT/c131072" # then use --contexts 131072 +``` + +## Strix bring-up evidence lane + +The 128 GiB or larger Apple Metal run remains an external verification gate for the pinned ds4 cross-runtime oracle. A Strix bring-up can complete first, but it must report `BRINGUP PASS`, never `TARGET PASS`. It does not replace complete byte-identical ds4 logits. + +The pinned `antirez/ds4@bd66c402070042bf0a79ad6ece8242de4c93680c` evidence anchors are: + +| Evidence | SHA-256 | Limitation | +|---|---|---| +| `tests/test-vectors/README.md` | `0e59b2f2832bed8af0a91e6ff20962debf964cd2d1d141c086e52cfcc995a1c3` | Official-vector provenance and limitations | +| `tests/test-vectors/flash-0731/manifest.json` | `ebf237a5660a6851fb8085e77f532901a9d758208b25d7ed5af0b7af4b28f91b` | Official API provenance | +| `tests/test-vectors/flash-0731/official.vec` | `77ae699889bfaf1348768dcbe7ea2c72279ae86abb10470d3e1b08cd1fd82a83` | Official selected-token/top-logprob slice, not full logits | +| `tests/test-vectors/flash-0731/local-golden.vec` | `23d942ff3b9bb2a3f82927d11aa3ed1461e1f302071e788d0d95a5c165e47d3b` | Local tolerant top-64 drift anchor, not exact | +| `tests/test_engram.c` | `198a561d981f62518a9d28035480a7e220b99c156cde6d248b8baabd684cc74b` | Model-free Engram oracle | +| `ds4_engram.c` | `2b6ca468510ebf45ee298a905525bc7234dacad9a384bf2011eba19ba2c0bdf7` | Engram implementation under test | +| `ds4_engram.h` | `f84a264e0fe199d23a6f0c56fbbd19e222adc7009eed13c185af6403f68f7f5c` | Engram schema | +| `tests/test_deepseek41_metal.c` | `9197c2f9d65b380ce25be5334991e4bfaf40e82e6708e64f2b9329552412d28c` | Synthetic exact candidate/top-k oracle; source anchor only on Strix | +| `tests/test_deepseek41_graph.c` | `6dc786f831c93ae7f5aa56e7518f67657125f7c0fd35cead3c646eff3d3f9e09` | Synthetic routing and graph oracle | +| `tests/test_deepseek41_prefill.c` | `452774b9332d393822d84288eeb2d71ca1fb25f1ba30d20a49160d1787de734f` | Prefill boundary fixture | +| `tests/test_deepseek41_manifest.py` | `2d7aa1fc93805d9c97839c5de9eccc856628f13e6913c3bbc3f0c99b0827aeae` | Model manifest/schema oracle | +| `tests/test_deepseek41_conversion.py` | `a40b83062a9b91338773addd77fe62650296f4de607057cb4fd1329287f347b5c` | Conversion/schema fixture | +| `tests/test_deepseek41_gguf.c` | `8f41e049d5ec179c38a1a00ac61712db0306902f0ef74bcdb989d8993316ff35` | GGUF schema oracle | +| `gguf-tools/deepseek41_metadata.py` | `39300bbd504165b97de017edd72563377f50b8a5511ca7478252a86f31a0009b` | Conversion metadata source | +| `ds4.c` | `1776dbfed177ea14f3ce6cac1d8d0b1c1b44dfff2c2663769a9a5634aeec34e7` | Runtime schema source | + +Verify these files from the pinned checkout before using their results. Do not copy an unpublished exporter or depend on a private ds4 remote. + +```sh +python3 tools/deepseek-v41-trace/verify_ds4_anchors.py \ + --checkout /home/papa/src/ds4-v41 +``` + +`run_matrix.py --llama-only` captures all four repository corpora without claiming cross-runtime success. Use the matrix command above, add `--llama-only`, and omit `--ds4-runner`, `--ds4-checkout`, `--ds4-exporter`, and `--ds4-exporter-sha256`. Run the final integration build twice with separate output directories, then run an equivalently instrumented immutable base build once. Keep every run under its own canonical watchdog invocation and use the exact ubatch/cache/ROCm arguments above. + +For each case (`correctness-prose-c32768-ub32`, `correctness-code-c32768-ub32`, `correctness-structured-c32768-ub32`, and `correctness-numeric-c32768-ub32`), require both comparisons: + +```sh +python3 tools/deepseek-v41-trace/trace_format.py compare-local self-consistency \ + "$RUN_A/llama/$CASE" "$RUN_B/llama/$CASE" \ + --report "$REPORTS/$CASE-self.json" + +python3 tools/deepseek-v41-trace/trace_format.py compare-local base-regression \ + "$BASE_RUN/llama/$CASE" "$RUN_A/llama/$CASE" \ + --report "$REPORTS/$CASE-base.json" +``` + +Both commands compare every exact trace component, including complete prefill/decode logits, tokens, Engram rows, original expert IDs and weights, raw/compressed attention sources, and candidate propagation. `base-regression` also requires the base trace revision to equal the integrated trace's attested oracle revision. The report includes `cross_runtime_status: "INCOMPLETE"` even when it returns `BRINGUP PASS`. + +The pinned ds4 evidence commands are: + +```sh +git -C /home/papa/src/ds4-v41 diff --quiet +git -C /home/papa/src/ds4-v41 diff --cached --quiet +test "$(git -C /home/papa/src/ds4-v41 rev-parse HEAD)" = bd66c402070042bf0a79ad6ece8242de4c93680c +/home/papa/src/ds4-v41/tests/test_engram +DS4_TEST_MODEL="$MODEL" \ +DS4_TEST_VECTOR_FILE=/home/papa/src/ds4-v41/tests/test-vectors/flash-0731/official.vec \ + /home/papa/src/ds4-v41/ds4_test --logprob-vectors +``` -`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the candidate revision, full-graph base revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass `--contexts 32768 65536 98304 131072` only after the 32K target passes. +Capture the exact commands, executable hashes, stdout/stderr hashes, exit status, and watchdog artifacts. The official-vector result and pinned fixtures are supporting bring-up evidence only. The first `TARGET PASS` still requires the external pinned ds4 exporter and the full cross-runtime trace comparison. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 7aaa219890b4..4923fd5f3c3b 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -7,12 +7,14 @@ extern "C" { #include "hash/sha256/sha256.h" } #include "llama.h" +#include "llama-ext.h" #include "trace-components.h" #include #include #include +#include #include #include #include @@ -26,12 +28,25 @@ extern "C" { #include #include #include +#include #include +#if defined(__linux__) +#include +#include +#include +#include +#include +#endif + namespace fs = std::filesystem; using json = nlohmann::ordered_json; static constexpr int TRACE_VERSION = 1; +#if defined(__linux__) +static constexpr const char * WATCHDOG_SCRIPT_SHA256 = + "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f"; +#endif static std::string sha256_hex(const unsigned char digest[SHA256_DIGEST_SIZE]) { std::ostringstream stream; @@ -107,7 +122,9 @@ static std::string required_environment(const char * name) { } #if defined(__linux__) -static uint64_t proc_start_time_ticks(int64_t pid) { +static constexpr uint64_t DSV41_GIB = UINT64_C(1024)*1024*1024; + +static std::pair proc_identity(int64_t pid) { const std::vector bytes = read_file("/proc/" + std::to_string(pid) + "/stat"); const std::string stat(bytes.begin(), bytes.end()); const size_t command_end = stat.rfind(')'); @@ -116,45 +133,216 @@ static uint64_t proc_start_time_ticks(int64_t pid) { } std::istringstream fields(stat.substr(command_end + 2)); std::string value; + int64_t parent = 0; for (int field = 3; field <= 22; ++field) { if (!(fields >> value)) { throw std::runtime_error("watchdog process stat is truncated"); } + if (field == 4) { + parent = std::stoll(value); + } } - return std::stoull(value); + return { parent, std::stoull(value) }; +} + +static bool process_is_descendant(int64_t pid, int64_t ancestor) { + std::vector seen; + while (pid > 1 && std::find(seen.begin(), seen.end(), pid) == seen.end()) { + if (pid == ancestor) { + return true; + } + seen.push_back(pid); + pid = proc_identity(pid).first; + } + return false; } static void validate_watchdog(const json & data) { - const int64_t pid = data.value("pid", INT64_C(0)); + const int64_t pid = data.value("watchdog_pid", INT64_C(0)); + const int64_t guardian_pid = data.value("guardian_pid", INT64_C(0)); + const int64_t child_pid = data.value("child_pid", INT64_C(0)); + const int64_t child_pgid = data.value("child_process_group_id", INT64_C(0)); if (pid <= 1 || !fs::exists("/proc/" + std::to_string(pid))) { throw std::runtime_error("watchdog process is not running"); } - if (proc_start_time_ticks(pid) != data.value("start_time_ticks", UINT64_C(0))) { + if (data.value("format", "") != "strix-memory-watchdog-lease" || data.value("version", 0) != 2) { + throw std::runtime_error("watchdog lease format is invalid"); + } + if (data.value("soft_bytes", UINT64_C(0)) != 116*DSV41_GIB || + data.value("emergency_bytes", UINT64_C(0)) != 118*DSV41_GIB || + data.value("strict_ceiling_bytes", UINT64_C(0)) != 120*DSV41_GIB || + data.value("grace_seconds", 0.0) != 30.0 || + data.value("sample_interval_seconds", 0.0) != 1.0 || + data.value("procfs_root", "") != "/proc") { + throw std::runtime_error("watchdog execution policy is invalid"); + } + if (guardian_pid <= 1 || child_pid <= 1 || child_pgid <= 1 || getpgrp() != child_pgid || + !process_is_descendant(getpid(), child_pid)) { + throw std::runtime_error("trace exporter is outside the watchdog-monitored process group"); + } + if (proc_identity(guardian_pid).first != pid || + proc_identity(child_pid).first != guardian_pid || + getpgid(guardian_pid) != child_pgid || + getpgid(child_pid) != child_pgid || + child_pgid != guardian_pid) { + throw std::runtime_error("watchdog guardian or child process identity is invalid"); + } + if (proc_identity(pid).second != data.value("watchdog_start_time_ticks", UINT64_C(0))) { throw std::runtime_error("watchdog process start time changed"); } const std::vector command = read_file("/proc/" + std::to_string(pid) + "/cmdline"); - if (sha256_data(command.data(), command.size()) != data.value("command_sha256", "")) { + if (sha256_data(command.data(), command.size()) != data.value("watchdog_command_sha256", "")) { throw std::runtime_error("watchdog process command changed"); } + const fs::path executable_path = data.value("watchdog_executable_path", ""); + if (executable_path.empty() || + fs::canonical("/proc/" + std::to_string(pid) + "/exe") != fs::canonical(executable_path)) { + throw std::runtime_error("watchdog executable identity changed"); + } + const fs::path script_path = data.value("watchdog_script_path", ""); + if (script_path.empty() || data.value("watchdog_revision", "") != + "778db6f50eae04e6c232c69b9575bdbd0747962b" || + data.value("watchdog_script_sha256", "") != WATCHDOG_SCRIPT_SHA256 || + sha256_file(script_path) != WATCHDOG_SCRIPT_SHA256) { + throw std::runtime_error("watchdog script identity changed"); + } + std::vector watchdog_arguments; + size_t argument_start = 0; + while (argument_start < command.size()) { + const auto * begin = reinterpret_cast(command.data() + argument_start); + const size_t argument_size = std::char_traits::length(begin); + watchdog_arguments.emplace_back(begin, argument_size); + argument_start += argument_size + 1; + } + fs::path command_script; + if (watchdog_arguments.size() >= 2) { + command_script = watchdog_arguments[1]; + if (!command_script.is_absolute()) { + command_script = fs::canonical("/proc/" + std::to_string(pid) + "/cwd") / command_script; + } + } + if (watchdog_arguments.size() < 2 || fs::canonical(command_script) != fs::canonical(script_path)) { + throw std::runtime_error("watchdog script is not in executable argv position"); + } const fs::path heartbeat_path = data.value("heartbeat_path", ""); - const int64_t max_age = data.value("max_heartbeat_age_seconds", INT64_C(0)); + const double max_age = data.value("max_heartbeat_age_seconds", 0.0); if (heartbeat_path.empty() || max_age <= 0 || max_age > 30) { throw std::runtime_error("watchdog heartbeat configuration is invalid"); } - const std::vector heartbeat_bytes = read_file(heartbeat_path); - const std::string heartbeat_text(heartbeat_bytes.begin(), heartbeat_bytes.end()); - size_t parsed = 0; - const int64_t heartbeat = std::stoll(heartbeat_text, &parsed); - while (parsed < heartbeat_text.size() && std::isspace(static_cast(heartbeat_text[parsed]))) { - ++parsed; + if (fs::canonical(required_environment("STRIX_MEMORY_WATCHDOG_LEASE_PATH")) != + fs::canonical(fs::path(data.value("lease_path", ""))) || + fs::canonical(required_environment("STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH")) != + fs::canonical(heartbeat_path) || + fs::canonical(required_environment("STRIX_MEMORY_WATCHDOG_AUDIT_PATH")) != + fs::canonical(fs::path(data.value("audit_live_path", ""))) || + std::stod(required_environment("STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS")) != max_age) { + throw std::runtime_error("watchdog lease does not match the inherited environment"); + } + const json child_command = data.value("command", json::array()); + if (!child_command.is_array() || child_command.empty()) { + throw std::runtime_error("watchdog child command is invalid"); + } + for (const json & argument : child_command) { + if (!argument.is_string()) { + throw std::runtime_error("watchdog child command is invalid"); + } } - if (parsed != heartbeat_text.size()) { - throw std::runtime_error("watchdog heartbeat is invalid"); + const std::string child_command_json = child_command.dump(-1, ' ', true); + if (sha256_data( + reinterpret_cast(child_command_json.data()), + child_command_json.size()) != data.value("child_command_sha256", "")) { + throw std::runtime_error("watchdog child command SHA-256 is invalid"); } - const int64_t now = static_cast(std::time(nullptr)); - if (heartbeat <= 0 || heartbeat > now || now - heartbeat > max_age) { + const std::vector heartbeat_bytes = read_file(heartbeat_path); + json heartbeat; + try { + heartbeat = json::parse(heartbeat_bytes.begin(), heartbeat_bytes.end()); + } catch (const json::exception & error) { + throw std::runtime_error(std::string("watchdog heartbeat is invalid: ") + error.what()); + } + if (heartbeat.value("format", "") != "strix-memory-watchdog-heartbeat" || + heartbeat.value("version", 0) != 2 || + heartbeat.value("lease_id", "") != data.value("lease_id", "") || + heartbeat.value("sequence", INT64_C(-1)) < 0 || + heartbeat.value("state", "") != "active" || + heartbeat.value("updated_at", "").empty() || + heartbeat.value("watchdog_pid", INT64_C(0)) != pid || + heartbeat.value("watchdog_start_time_ticks", UINT64_C(0)) != + data.value("watchdog_start_time_ticks", UINT64_C(0)) || + heartbeat.value("child_pid", INT64_C(0)) != child_pid || + heartbeat.value("child_process_group_id", INT64_C(0)) != child_pgid) { + throw std::runtime_error("watchdog heartbeat identity is invalid"); + } + const json heartbeat_sample = heartbeat.value("sample", json::object()); + const std::string audit_record_sha256 = heartbeat_sample.value("audit_record_sha256", ""); + if (audit_record_sha256.size() != 64) { + throw std::runtime_error("watchdog heartbeat audit identity is invalid"); + } + const uint64_t updated_monotonic_ns = heartbeat.value("updated_monotonic_ns", UINT64_C(0)); + struct timespec now; + if (updated_monotonic_ns == 0 || clock_gettime(CLOCK_MONOTONIC, &now) != 0) { + throw std::runtime_error("watchdog heartbeat monotonic timestamp is invalid"); + } + const uint64_t now_monotonic_ns = + static_cast(now.tv_sec)*UINT64_C(1000000000) + static_cast(now.tv_nsec); + const uint64_t max_age_ns = static_cast(max_age*1000000000.0); + if (updated_monotonic_ns > now_monotonic_ns || + now_monotonic_ns - updated_monotonic_ns > max_age_ns) { throw std::runtime_error("watchdog heartbeat is stale"); } + const fs::path audit_path = data.value("audit_live_path", ""); + if (audit_path.empty()) { + throw std::runtime_error("watchdog audit path is invalid"); + } + struct stat audit_stat; + struct stat descriptor_stat; + const int audit_fd = data.value("audit_fd", -1); + const fs::path descriptor_path = + "/proc/" + std::to_string(pid) + "/fd/" + std::to_string(audit_fd); + if (audit_fd < 0 || lstat(audit_path.c_str(), &audit_stat) != 0 || + stat(descriptor_path.c_str(), &descriptor_stat) != 0 || + !S_ISREG(audit_stat.st_mode) || + static_cast(audit_stat.st_dev) != data.value("audit_device", UINT64_C(0)) || + static_cast(audit_stat.st_ino) != data.value("audit_inode", UINT64_C(0)) || + static_cast(descriptor_stat.st_dev) != data.value("audit_device", UINT64_C(0)) || + static_cast(descriptor_stat.st_ino) != data.value("audit_inode", UINT64_C(0)) || + audit_stat.st_uid != getuid() || + static_cast(audit_stat.st_uid) != data.value("audit_uid", UINT64_C(0)) || + (audit_stat.st_mode & 0777) != 0600 || + data.value("audit_mode", UINT64_C(0)) != 0600) { + throw std::runtime_error("watchdog persistent audit identity changed"); + } + const int local_audit_fd = open(audit_path.c_str(), O_RDONLY | O_NOFOLLOW); + if (local_audit_fd < 0) { + throw std::runtime_error("cannot open watchdog persistent audit"); + } + const int lock_result = flock(local_audit_fd, LOCK_EX | LOCK_NB); + if (lock_result == 0) { + flock(local_audit_fd, LOCK_UN); + close(local_audit_fd); + throw std::runtime_error("watchdog does not hold the persistent audit lock"); + } + if (errno != EWOULDBLOCK && errno != EAGAIN) { + close(local_audit_fd); + throw std::runtime_error("cannot inspect watchdog persistent audit lock"); + } + close(local_audit_fd); + std::ifstream audit_stream(audit_path); + if (!audit_stream) { + throw std::runtime_error("cannot read watchdog persistent audit"); + } + std::string audit_line; + bool found_audit_record = false; + while (std::getline(audit_stream, audit_line)) { + audit_line.push_back('\n'); + if (sha256_data(audit_line.data(), audit_line.size()) == audit_record_sha256) { + found_audit_record = true; + break; + } + } + if (!found_audit_record) { + throw std::runtime_error("watchdog heartbeat audit record is missing"); + } } #endif @@ -171,6 +359,9 @@ static json audit_reference(const char * environment_name, const char * expected if (audit.value("kind", "") != expected_kind) { throw std::runtime_error(std::string("audit kind mismatch for ") + expected_kind); } + if (audit.value("environment", json::object()).value("HIP_LAUNCH_BLOCKING", "") != "1") { + throw std::runtime_error(std::string("audit environment mismatch for ") + expected_kind); + } const int64_t created = audit.value("created_unix", INT64_C(0)); const int64_t now = static_cast(std::time(nullptr)); if (created <= 0 || now < created || now - created > 300) { @@ -458,6 +649,9 @@ int main(int argc, char ** argv) { require_nvme_path(params.model.path, "model"); require_nvme_path(params.prompt_file, "prompt"); require_nvme_path(params.out_file, "trace output"); + if (required_environment("HIP_LAUNCH_BLOCKING") != "1") { + throw std::runtime_error("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs"); + } const json memory_audit = audit_reference("DSV41_TRACE_MEMORY_AUDIT", "memory"); const json swap_audit = audit_reference("DSV41_TRACE_SWAP_AUDIT", "swap"); const json watchdog_audit = audit_reference("DSV41_TRACE_WATCHDOG_AUDIT", "watchdog"); @@ -482,6 +676,13 @@ int main(int argc, char ** argv) { if (model_architecture(model) != "deepseek41") { throw std::runtime_error("trace tool requires general.architecture=deepseek41"); } + std::vector model_devices; + for (int32_t index = 0; index < llama_model_n_devices(model); ++index) { + model_devices.emplace_back(ggml_backend_dev_name(llama_model_get_device(model, index))); + } + if (model_devices != std::vector{"ROCm0"}) { + throw std::runtime_error("trace tool requires the loaded model to use only ROCm0"); + } const llama_vocab * vocab = llama_model_get_vocab(model); const bool add_bos = llama_vocab_get_add_bos(vocab); const std::vector tokens = common_tokenize(ctx, params.prompt, add_bos, true); @@ -527,6 +728,7 @@ int main(int argc, char ** argv) { {"context", llama_n_ctx(ctx)}, {"batch", params.n_batch}, {"ubatch", params.n_ubatch}, + {"device", model_devices[0]}, {"decode_steps", params.n_predict}, {"kv_type_k", ggml_type_name(params.cache_type_k)}, {"kv_type_v", ggml_type_name(params.cache_type_v)}, @@ -548,6 +750,8 @@ int main(int argc, char ** argv) { {"candidate_topk_blocks", 2048}, {"candidate_block_size", 8}, {"index_top_k", 512}, + {"raw_attention_layers", {0, 1}}, + {"raw_attention_width", 128}, {"candidate_propagation_layers", {24, 28, 32, 36}}, }}, }}, diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index c66be6627ee8..ed232ca3e2d3 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -2,15 +2,32 @@ import json import hashlib +import importlib.util import os import re -import shutil +import sys import time +from datetime import datetime from pathlib import Path +from typing import Callable FORBIDDEN_ROOT = Path("/mnt/bigspace") SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 -MAX_WATCHDOG_HEARTBEAT_AGE = 30 +WATCHDOG_EMERGENCY_LIMIT = 118 * 1024 * 1024 * 1024 +STRICT_MEMORY_LIMIT = 120 * 1024 * 1024 * 1024 +MAX_WATCHDOG_HEARTBEAT_AGE = 30.0 +WATCHDOG_LEASE_FORMAT = "strix-memory-watchdog-lease" +WATCHDOG_HEARTBEAT_FORMAT = "strix-memory-watchdog-heartbeat" +WATCHDOG_VERSION = 2 +WATCHDOG_STARTUP_TIMEOUT_SECONDS = 5.0 +WATCHDOG_LEASE_ENV = "STRIX_MEMORY_WATCHDOG_LEASE_PATH" +WATCHDOG_HEARTBEAT_ENV = "STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH" +WATCHDOG_AUDIT_ENV = "STRIX_MEMORY_WATCHDOG_AUDIT_PATH" +WATCHDOG_MAX_AGE_ENV = "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS" +WATCHDOG_REVISION = "778db6f50eae04e6c232c69b9575bdbd0747962b" +WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f" +APPROVED_WATCHDOGS = {WATCHDOG_SCRIPT_SHA256: WATCHDOG_REVISION} +_WATCHDOG_GUARD = None class PreflightError(RuntimeError): @@ -30,6 +47,27 @@ def require_nvme_path(path: Path, label: str) -> Path: raise PreflightError(f"{label} must not use rotational storage: {path}") +def safe_trace_path(root: Path, relative: Path | str) -> Path: + root = root.expanduser().absolute() + if root.is_symlink(): + raise PreflightError("trace output root must not be a symlink") + root = require_nvme_path(root, "trace output") + relative = Path(relative) + if relative.is_absolute() or ".." in relative.parts: + raise PreflightError(f"trace output path is outside the bundle: {relative}") + candidate = root + for part in relative.parts: + candidate = candidate / part + if candidate.is_symlink(): + raise PreflightError(f"trace output path must not use symlinks: {relative}") + try: + candidate.resolve().relative_to(root) + except ValueError as error: + raise PreflightError(f"trace output path is outside the bundle: {relative}") from error + require_nvme_path(candidate, "trace output") + return candidate + + def read_proc_lines(path: Path) -> list[str]: try: return path.read_text(encoding="ascii").splitlines() @@ -88,63 +126,460 @@ def proc_start_time_ticks(stat: str) -> int: return int(fields[19]) -def read_heartbeat(path: Path, max_age_seconds: int) -> int: +def proc_parent_pid(stat: str) -> int: + command_end = stat.rfind(")") + if command_end < 0: + raise PreflightError("process stat is invalid") + fields = stat[command_end + 2:].split() + if len(fields) < 2: + raise PreflightError("process stat is truncated") + return int(fields[1]) + + +def is_descendant(pid: int, ancestor_pid: int, procfs_root: Path) -> bool: + seen = set() + while pid > 1 and pid not in seen: + if pid == ancestor_pid: + return True + seen.add(pid) + try: + pid = proc_parent_pid((procfs_root / str(pid) / "stat").read_text(encoding="ascii")) + except (OSError, ValueError): + return False + return False + + +def read_heartbeat( + path: Path, + max_age_seconds: float, + *, + lease_id: str, + watchdog_pid: int, + watchdog_start_time_ticks: int, + child_pid: int, + child_pgid: int, + now: int | None = None, + monotonic_ns: Callable[[], int] = time.monotonic_ns) -> int: try: - heartbeat = int(path.read_text(encoding="ascii").strip()) - except (OSError, ValueError) as error: + record = json.loads(path.read_text(encoding="ascii")) + updated = datetime.fromisoformat(str(record["updated_at"]).replace("Z", "+00:00")) + heartbeat = int(updated.timestamp()) + except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: raise PreflightError(f"watchdog heartbeat is invalid: {error}") from error - now = int(time.time()) - if heartbeat <= 0 or heartbeat > now or now - heartbeat > max_age_seconds: + if record.get("format") != WATCHDOG_HEARTBEAT_FORMAT or record.get("version") != WATCHDOG_VERSION: + raise PreflightError("watchdog heartbeat format is invalid") + if record.get("lease_id") != lease_id or record.get("state") != "active": + raise PreflightError("watchdog heartbeat lease identity or state is invalid") + if not isinstance(record.get("sequence"), int) or record["sequence"] < 0: + raise PreflightError("watchdog heartbeat sequence is invalid") + if not isinstance(record.get("updated_monotonic_ns"), int) or record["updated_monotonic_ns"] <= 0: + raise PreflightError("watchdog heartbeat monotonic timestamp is invalid") + if record.get("watchdog_pid") != watchdog_pid or ( + record.get("watchdog_start_time_ticks") != watchdog_start_time_ticks): + raise PreflightError("watchdog heartbeat owner does not match the lease") + if record.get("child_pid") != child_pid or record.get("child_process_group_id") != child_pgid: + raise PreflightError("watchdog heartbeat child identity does not match the lease") + age_ns = monotonic_ns() - record["updated_monotonic_ns"] + if age_ns < 0 or age_ns > int(max_age_seconds * 1_000_000_000): raise PreflightError("watchdog heartbeat is stale") + now = int(time.time()) if now is None else now + if heartbeat <= 0 or heartbeat > now: + raise PreflightError("watchdog heartbeat wall-clock timestamp is invalid") return heartbeat -def watchdog_audit(pid_file: Path) -> dict[str, object]: +def _read_json_with_retry( + path: Path, + *, + timeout_seconds: float, + monotonic: Callable[[], float], + sleeper: Callable[[float], None]) -> dict[str, object]: + deadline = monotonic() + timeout_seconds + last_error: Exception | None = None + while True: + try: + record = json.loads(path.read_text(encoding="ascii")) + if not isinstance(record, dict): + raise ValueError("record is not an object") + return record + except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: + last_error = error + if monotonic() >= deadline: + raise PreflightError(f"watchdog lease did not become ready: {last_error}") from last_error + sleeper(0.05) + + +def _read_watchdog_events(path: Path) -> list[dict[str, object]]: + try: + lines = path.read_text(encoding="ascii").splitlines() + except OSError as error: + raise PreflightError(f"cannot read watchdog audit: {error}") from error + events = [] + for line_number, line in enumerate(lines, start=1): + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise PreflightError(f"watchdog audit line {line_number} is invalid: {error}") from error + if not isinstance(event, dict) or not isinstance(event.get("event"), str): + raise PreflightError(f"watchdog audit line {line_number} is not an event") + events.append(event) + if not events: + raise PreflightError("watchdog audit is empty") + return events + + +def _read_watchdog_startup_events( + path: Path, + *, + timeout_seconds: float, + monotonic: Callable[[], float], + sleeper: Callable[[float], None]) -> list[dict[str, object]]: + deadline = monotonic() + timeout_seconds + last_error: Exception | None = None + while True: + try: + events = _read_watchdog_events(path) + if any(event.get("event") == "preflight" for event in events) and ( + any(event.get("event") == "child_started" for event in events)): + return events + last_error = PreflightError("watchdog audit lacks preflight or child_started evidence") + except PreflightError as error: + last_error = error + if monotonic() >= deadline: + assert last_error is not None + raise PreflightError(f"watchdog audit did not become ready: {last_error}") from last_error + sleeper(0.05) + + +def _load_watchdog_module(script: Path) -> object: + spec = importlib.util.spec_from_file_location("dsv41_strix_memory_watchdog", script) + if spec is None or spec.loader is None: + raise PreflightError(f"cannot load canonical watchdog module: {script}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError, RuntimeError, SyntaxError) as error: + raise PreflightError(f"cannot load canonical watchdog module: {error}") from error + return module + + +def _canonical_watchdog_audit( + repo: Path, + *, + environment: dict[str, str] | os._Environ[str], + current_pid: int | None, + timeout_seconds: float, + monotonic: Callable[[], float], + sleeper: Callable[[float], None], + watchdog_module: object | None) -> dict[str, object]: + global _WATCHDOG_GUARD + try: + lease_path = require_nvme_path(Path(environment[WATCHDOG_LEASE_ENV]), "watchdog lease") + heartbeat_path = require_nvme_path(Path(environment[WATCHDOG_HEARTBEAT_ENV]), "watchdog heartbeat") + audit_path = require_nvme_path(Path(environment[WATCHDOG_AUDIT_ENV]), "watchdog audit") + max_age_seconds = float(environment[WATCHDOG_MAX_AGE_ENV]) + except (KeyError, ValueError) as error: + raise PreflightError(f"canonical watchdog environment is incomplete: {error}") from error + if max_age_seconds <= 0 or max_age_seconds > MAX_WATCHDOG_HEARTBEAT_AGE: + raise PreflightError(f"watchdog heartbeat age must be within 1..{MAX_WATCHDOG_HEARTBEAT_AGE} seconds") + expected_script = resolved(repo / "scripts" / "strix_memory_watchdog.py") + if not expected_script.is_file(): + raise PreflightError(f"canonical watchdog script is missing: {expected_script}") + script_sha256 = sha256_bytes(expected_script.read_bytes()) + watchdog_revision = APPROVED_WATCHDOGS.get(script_sha256) + if watchdog_revision is None: + raise PreflightError( + "no independently reviewed watchdog revision is approved for correctness execution") + module = watchdog_module or _load_watchdog_module(expected_script) + validator = getattr(module, "validate_active_lease", None) + validation_error = getattr(module, "LeaseValidationError", RuntimeError) + guard = getattr(module, "start_process_group_lease_guard", None) + if not callable(validator) or not isinstance(validation_error, type): + raise PreflightError("canonical watchdog module lacks the lease validation API") + process_id = os.getpid() if current_pid is None else current_pid + try: + lease = validator( + lease_path, + expected_script_path=expected_script, + expected_soft_bytes=SOFT_MEMORY_LIMIT, + expected_emergency_bytes=WATCHDOG_EMERGENCY_LIMIT, + expected_procfs_root=Path("/proc"), + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=max_age_seconds, + current_process_id=process_id, + process_procfs_root=Path("/proc"), + ) + if lease.get("child_pid") == process_id and _WATCHDOG_GUARD is None: + if not callable(guard): + raise PreflightError("canonical watchdog module lacks the process-group lease guard") + _WATCHDOG_GUARD = guard( + expected_script, + startup_timeout_seconds=timeout_seconds, + expected_procfs_root=Path("/proc"), + process_procfs_root=Path("/proc"), + ) + except validation_error as error: + raise PreflightError(f"canonical watchdog lease is invalid: {error}") from error + if not isinstance(lease, dict): + raise PreflightError("canonical watchdog lease validator returned invalid data") + if lease.get("grace_seconds") != 30.0 or lease.get("sample_interval_seconds") != 1.0: + raise PreflightError("canonical watchdog timing policy is invalid") + events = _read_watchdog_startup_events( + audit_path, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + ) + try: + heartbeat_record = json.loads(heartbeat_path.read_text(encoding="ascii")) + updated = datetime.fromisoformat(str(heartbeat_record["updated_at"]).replace("Z", "+00:00")) + heartbeat_unix = int(updated.timestamp()) + except (OSError, UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + raise PreflightError(f"canonical watchdog heartbeat is invalid: {error}") from error + try: + audit_sha256 = sha256_bytes(audit_path.read_bytes()) + except OSError as error: + raise PreflightError(f"cannot hash canonical watchdog audit: {error}") from error + watchdog_command = lease.get("watchdog_command") + if not isinstance(watchdog_command, str) or not watchdog_command: + try: + command_bytes = ( + Path("/proc") / str(lease["watchdog_pid"]) / "cmdline").read_bytes() + except (OSError, KeyError) as error: + raise PreflightError(f"cannot read canonical watchdog command: {error}") from error + watchdog_command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() + if not watchdog_command: + raise PreflightError("canonical watchdog command is empty") + result = dict(lease) + result.update({ + "watchdog_revision": watchdog_revision, + "lease_path": str(lease_path), + "watchdog_command": watchdog_command, + "heartbeat_unix": heartbeat_unix, + "audit_path": str(audit_path), + "audit_sha256": audit_sha256, + "audit_event_count": len(events), + }) + return result + + +def watchdog_audit( + repo: Path, + *, + environment: dict[str, str] | os._Environ[str] = os.environ, + procfs_root: Path = Path("/proc"), + current_pid: int | None = None, + current_pgid: int | None = None, + getpgid: Callable[[int], int] = os.getpgid, + now: int | None = None, + monotonic_ns: Callable[[], int] = time.monotonic_ns, + timeout_seconds: float = WATCHDOG_STARTUP_TIMEOUT_SECONDS, + monotonic: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, + watchdog_module: object | None = None) -> dict[str, object]: + if watchdog_module is not None or ( + procfs_root == Path("/proc") and current_pid is None and current_pgid is None): + return _canonical_watchdog_audit( + repo, + environment=environment, + current_pid=current_pid, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + watchdog_module=watchdog_module, + ) + try: + pid_file = Path(environment[WATCHDOG_LEASE_ENV]) + environment_heartbeat = require_nvme_path( + Path(environment[WATCHDOG_HEARTBEAT_ENV]), "watchdog heartbeat") + environment_audit = require_nvme_path( + Path(environment[WATCHDOG_AUDIT_ENV]), "watchdog audit") + environment_max_age = float(environment[WATCHDOG_MAX_AGE_ENV]) + except (KeyError, ValueError) as error: + raise PreflightError(f"canonical watchdog environment is incomplete: {error}") from error pid_file = require_nvme_path(pid_file, "watchdog lease") + repo = resolved(repo) + expected_script = resolved(repo / "scripts" / "strix_memory_watchdog.py") + if not expected_script.is_file(): + raise PreflightError(f"canonical watchdog script is missing: {expected_script}") + expected_script_sha256 = sha256_bytes(expected_script.read_bytes()) + lease = _read_json_with_retry( + pid_file, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + ) try: - lease = json.loads(pid_file.read_text(encoding="ascii")) - pid = int(lease["pid"]) - expected_start = int(lease["start_time_ticks"]) - expected_command_sha256 = str(lease["command_sha256"]) + pid = int(lease["watchdog_pid"]) + expected_start = int(lease["watchdog_start_time_ticks"]) + expected_command_sha256 = str(lease["watchdog_command_sha256"]) + child_pid = int(lease["child_pid"]) + child_pgid = int(lease["child_process_group_id"]) + script_path = resolved(Path(str(lease["watchdog_script_path"]))) + script_sha256 = str(lease["watchdog_script_sha256"]) + lease_id = str(lease["lease_id"]) + soft_bytes = int(lease["soft_bytes"]) + emergency_bytes = int(lease["emergency_bytes"]) + strict_bytes = int(lease["strict_ceiling_bytes"]) + procfs_path = str(lease["procfs_root"]) heartbeat_path = require_nvme_path(Path(lease["heartbeat_path"]), "watchdog heartbeat") - max_age_seconds = int(lease.get("max_heartbeat_age_seconds", MAX_WATCHDOG_HEARTBEAT_AGE)) - except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + audit_path = require_nvme_path(Path(lease["audit_path"]), "watchdog audit") + max_age_seconds = float(lease.get("max_heartbeat_age_seconds", MAX_WATCHDOG_HEARTBEAT_AGE)) + except (OSError, ValueError, TypeError, KeyError) as error: raise PreflightError(f"watchdog lease is invalid: {error}") from error + if lease.get("format") != WATCHDOG_LEASE_FORMAT or lease.get("version") != WATCHDOG_VERSION: + raise PreflightError("watchdog lease format is invalid") + if lease.get("state") != "active" or re.fullmatch(r"[0-9a-f]{32,64}", lease_id) is None: + raise PreflightError("watchdog lease identity or state is invalid") + if script_path != expected_script or script_sha256 != expected_script_sha256: + raise PreflightError("watchdog script identity does not match the candidate repository") + if soft_bytes != SOFT_MEMORY_LIMIT or emergency_bytes != WATCHDOG_EMERGENCY_LIMIT or ( + strict_bytes != STRICT_MEMORY_LIMIT): + raise PreflightError("watchdog memory thresholds are invalid") + if procfs_path != "/proc": + raise PreflightError("watchdog procfs root must be /proc") + if heartbeat_path != environment_heartbeat or audit_path != environment_audit or ( + max_age_seconds != environment_max_age): + raise PreflightError("watchdog lease paths or heartbeat age do not match the inherited environment") if re.fullmatch(r"[0-9a-f]{64}", expected_command_sha256) is None: raise PreflightError("watchdog lease command SHA-256 is invalid") if max_age_seconds <= 0 or max_age_seconds > MAX_WATCHDOG_HEARTBEAT_AGE: raise PreflightError(f"watchdog heartbeat age must be within 1..{MAX_WATCHDOG_HEARTBEAT_AGE} seconds") - if pid <= 1 or not Path(f"/proc/{pid}").exists(): + if pid <= 1 or child_pid <= 1 or child_pgid <= 1: + raise PreflightError("watchdog or monitored child identity is invalid") + if current_pgid is None: + current_pgid = os.getpgrp() + if current_pid is None: + current_pid = os.getpid() + if current_pgid != child_pgid: + raise PreflightError("current process is outside the watchdog-monitored process group") + if not is_descendant(current_pid, child_pid, procfs_root): + raise PreflightError("current process is not a descendant of the watchdog-monitored child") + try: + child_parent = proc_parent_pid( + (procfs_root / str(child_pid) / "stat").read_text(encoding="ascii")) + if child_parent != pid or getpgid(child_pid) != child_pgid or child_pgid != child_pid: + raise PreflightError("watchdog child process group does not match the lease") + except (OSError, ValueError) as error: + raise PreflightError(f"cannot inspect watchdog child process group: {error}") from error + if not (procfs_root / str(pid)).exists(): raise PreflightError(f"watchdog process {pid} is not running") try: - command_bytes = Path(f"/proc/{pid}/cmdline").read_bytes() - start_time_ticks = proc_start_time_ticks(Path(f"/proc/{pid}/stat").read_text(encoding="ascii")) + command_bytes = (procfs_root / str(pid) / "cmdline").read_bytes() + start_time_ticks = proc_start_time_ticks( + (procfs_root / str(pid) / "stat").read_text(encoding="ascii")) except (OSError, ValueError) as error: raise PreflightError(f"cannot inspect watchdog process {pid}: {error}") from error command_sha256 = sha256_bytes(command_bytes) if start_time_ticks != expected_start or command_sha256 != expected_command_sha256: raise PreflightError("watchdog process identity does not match its lease") - heartbeat = read_heartbeat(heartbeat_path, max_age_seconds) + command_parts = [part.decode("utf-8", "replace") for part in command_bytes.split(b"\0") if part] + try: + watchdog_cwd = (procfs_root / str(pid) / "cwd").resolve() + except OSError as error: + raise PreflightError(f"cannot inspect watchdog process working directory: {error}") from error + script_named = any( + resolved(Path(argument) if Path(argument).is_absolute() else watchdog_cwd / argument) == expected_script + for argument in command_parts + ) + if not script_named: + raise PreflightError("watchdog command does not execute the candidate repository script") + child_command = lease.get("command") + if not isinstance(child_command, list) or not child_command or ( + not all(isinstance(argument, str) for argument in child_command)): + raise PreflightError("watchdog child command is invalid") + child_command_sha256 = sha256_bytes( + json.dumps(child_command, ensure_ascii=True, separators=(",", ":")).encode("utf-8")) + if child_command_sha256 != lease.get("child_command_sha256"): + raise PreflightError("watchdog child command does not match the lease") + heartbeat = read_heartbeat( + heartbeat_path, + max_age_seconds, + lease_id=lease_id, + watchdog_pid=pid, + watchdog_start_time_ticks=start_time_ticks, + child_pid=child_pid, + child_pgid=child_pgid, + now=now, + monotonic_ns=monotonic_ns, + ) command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() if not command: raise PreflightError(f"watchdog process {pid} has no command line") + events = _read_watchdog_startup_events( + audit_path, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + ) + preflight = next((event for event in events if event.get("event") == "preflight"), None) + child_started = next((event for event in events if event.get("event") == "child_started"), None) + if preflight is None or child_started is None: + raise PreflightError("watchdog audit lacks preflight or child_started evidence") + if preflight.get("soft_bytes") != SOFT_MEMORY_LIMIT or ( + preflight.get("emergency_bytes") != WATCHDOG_EMERGENCY_LIMIT) or ( + preflight.get("strict_ceiling_bytes") != STRICT_MEMORY_LIMIT): + raise PreflightError("watchdog audit thresholds are invalid") + if preflight.get("swap_entries") != 0: + raise PreflightError("watchdog audit does not report zero swap") + if child_started.get("child_pid") != child_pid or ( + child_started.get("process_group_id") != child_pgid): + raise PreflightError("watchdog audit child identity does not match the lease") + if child_started.get("command") != lease.get("command"): + raise PreflightError("watchdog audit child command does not match the lease") return { - "pid": pid, - "pid_file": str(pid_file), - "start_time_ticks": start_time_ticks, - "command": command, - "command_sha256": command_sha256, + "format": WATCHDOG_LEASE_FORMAT, + "version": WATCHDOG_VERSION, + "lease_id": lease_id, + "lease_path": str(pid_file), + "watchdog_pid": pid, + "watchdog_start_time_ticks": start_time_ticks, + "watchdog_command": command, + "watchdog_command_sha256": command_sha256, + "watchdog_script_path": str(script_path), + "watchdog_script_sha256": script_sha256, + "soft_bytes": soft_bytes, + "emergency_bytes": emergency_bytes, + "strict_ceiling_bytes": strict_bytes, + "procfs_root": procfs_path, + "child_pid": child_pid, + "child_process_group_id": child_pgid, + "command": child_command, + "child_command_sha256": child_command_sha256, "heartbeat_path": str(heartbeat_path), "heartbeat_unix": heartbeat, "max_heartbeat_age_seconds": max_age_seconds, + "audit_path": str(audit_path), + "audit_sha256": sha256_bytes(audit_path.read_bytes()), + "audit_event_count": len(events), } -def matching_workloads(patterns: list[str]) -> list[dict[str, object]]: +def process_ancestry(pid: int, procfs_root: Path) -> set[int]: + ancestors = {pid} + while pid > 1: + try: + parent = proc_parent_pid((procfs_root / str(pid) / "stat").read_text(encoding="ascii")) + except (OSError, UnicodeError, ValueError): + break + if parent <= 1 or parent in ancestors: + break + ancestors.add(parent) + pid = parent + return ancestors + + +def matching_workloads( + patterns: list[str], + *, + procfs_root: Path = Path("/proc"), + current_pid: int | None = None) -> list[dict[str, object]]: matches = [] - excluded = {os.getpid(), os.getppid()} + excluded = process_ancestry(os.getpid() if current_pid is None else current_pid, procfs_root) lowered = [pattern.lower() for pattern in patterns if pattern] - for entry in Path("/proc").iterdir(): + for entry in procfs_root.iterdir(): if not entry.name.isdigit(): continue pid = int(entry.name) @@ -165,7 +600,7 @@ def run_preflight( model: Path, prompt: Path, output: Path, - watchdog_pid_file: Path, + repo: Path, busy_patterns: list[str], ) -> dict[str, object]: model = require_nvme_path(model, "model") @@ -175,10 +610,12 @@ def run_preflight( raise PreflightError(f"model is not a file: {model}") if not prompt.is_file(): raise PreflightError(f"prompt is not a file: {prompt}") + if os.environ.get("HIP_LAUNCH_BLOCKING") != "1": + raise PreflightError("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs") swap = swap_audit() if swap["enabled"]: raise PreflightError("swap is enabled; model execution is blocked") - watchdog = watchdog_audit(watchdog_pid_file) + watchdog = watchdog_audit(repo) workloads = matching_workloads(busy_patterns) if workloads: raise PreflightError("active model workload detected: " + json.dumps(workloads, ensure_ascii=True)) @@ -191,6 +628,7 @@ def run_preflight( "swap": swap, "watchdog": watchdog, "active_workloads": [], + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, } @@ -202,10 +640,39 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: result = {} for key in ("memory", "swap", "watchdog"): path = root / f"{key}.json" + data = audit[key] + if key == "watchdog": + data = dict(data) + live_audit_path = resolved(Path(str(data["audit_path"]))) + try: + audit_bytes = live_audit_path.read_bytes() + audit_text = audit_bytes.decode("ascii") + except (OSError, UnicodeError) as error: + raise PreflightError(f"cannot snapshot watchdog audit: {error}") from error + events = [] + for line_number, line in enumerate(audit_text.splitlines(), start=1): + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise PreflightError( + f"watchdog audit line {line_number} is invalid while snapshotting: {error}") from error + if not isinstance(event, dict) or not isinstance(event.get("event"), str): + raise PreflightError( + f"watchdog audit line {line_number} is not an event while snapshotting") + events.append(event) + if not events: + raise PreflightError("watchdog audit snapshot is empty") + snapshot = root / "watchdog-events.jsonl" + snapshot.write_bytes(audit_bytes) + data["audit_live_path"] = str(live_audit_path) + data["audit_path"] = str(snapshot) + data["audit_sha256"] = sha256_bytes(audit_bytes) + data["audit_event_count"] = len(events) value = { "created_unix": audit["created_unix"], "kind": key, - "data": audit[key], + "data": data, + "environment": audit["environment"], } path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") result[key] = str(path) @@ -215,24 +682,75 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: return result +def seal_audits(audits: dict[str, str]) -> dict[str, str]: + digests = {} + paths = [] + try: + for kind in ("memory", "swap", "watchdog"): + path = resolved(Path(audits[kind])) + paths.append(path) + data = path.read_bytes() + digests[kind] = sha256_bytes(data) + if kind == "watchdog": + record = json.loads(data.decode("ascii")) + jsonl_path = resolved(Path(record["data"]["audit_path"])) + paths.append(jsonl_path) + digests["watchdog_jsonl"] = sha256_bytes(jsonl_path.read_bytes()) + for path in paths: + path.chmod(0o444) + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError) as error: + raise PreflightError(f"cannot seal audit evidence: {error}") from error + return digests + + +def verify_sealed_audits(audits: dict[str, str], digests: dict[str, str]) -> None: + current = seal_audits(audits) + if current != digests: + raise PreflightError("preflight audit evidence changed during runtime execution") + + def embed_audits(trace_root: Path, phase: str, audits: dict[str, str]) -> dict[str, dict[str, object]]: - trace_root = resolved(trace_root) - embedded_root = trace_root / "audits" / phase + trace_root = safe_trace_path(trace_root, ".") + embedded_root = safe_trace_path(trace_root, Path("audits") / phase) embedded_root.mkdir(parents=True, exist_ok=True) result = {} for kind in ("memory", "swap", "watchdog"): source = resolved(Path(audits[kind])) data = source.read_bytes() + try: + record = json.loads(data.decode("ascii")) + except (UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot embed {kind} audit: {error}") from error + if kind == "watchdog": + try: + jsonl_source = resolved(Path(record["data"].pop("audit_path"))) + jsonl_data = jsonl_source.read_bytes() + except (OSError, TypeError, KeyError) as error: + raise PreflightError(f"cannot embed watchdog JSONL audit: {error}") from error + jsonl_digest = sha256_bytes(jsonl_data) + if record["data"].get("audit_sha256") != jsonl_digest: + raise PreflightError("watchdog JSONL audit SHA-256 changed before embedding") + jsonl_destination = safe_trace_path( + trace_root, Path("audits") / phase / f"{jsonl_digest}.jsonl") + if jsonl_destination.exists() and jsonl_destination.read_bytes() != jsonl_data: + raise PreflightError(f"content-addressed watchdog audit collision: {jsonl_destination}") + if not jsonl_destination.exists(): + jsonl_destination.write_bytes(jsonl_data) + record["data"]["audit"] = { + "path": f"audits/{phase}/{jsonl_digest}.jsonl", + "sha256": jsonl_digest, + "event_count": record["data"].pop("audit_event_count"), + } + data = (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") digest = sha256_bytes(data) - destination = embedded_root / f"{digest}.json" + destination = safe_trace_path(trace_root, Path("audits") / phase / f"{digest}.json") if destination.exists() and destination.read_bytes() != data: raise PreflightError(f"content-addressed audit collision: {destination}") if not destination.exists(): - shutil.copyfile(source, destination) + destination.write_bytes(data) try: - record = json.loads(data.decode("ascii")) created = int(record["created_unix"]) - except (UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + except (ValueError, TypeError, KeyError) as error: raise PreflightError(f"cannot embed {kind} audit: {error}") from error result[kind] = { "path": f"audits/{phase}/{digest}.json", @@ -243,8 +761,8 @@ def embed_audits(trace_root: Path, phase: str, audits: dict[str, str]) -> dict[s def bind_embedded_audits(trace_root: Path, audit_sets: dict[str, dict[str, str]]) -> None: - trace_root = resolved(trace_root) - manifest_path = trace_root / "manifest.json" + trace_root = safe_trace_path(trace_root, ".") + manifest_path = safe_trace_path(trace_root, "manifest.json") try: manifest = json.loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: @@ -301,8 +819,8 @@ def validate_prompt_provenance( def bind_prompt_provenance(trace_root: Path, provenance: dict[str, object]) -> None: - trace_root = resolved(trace_root) - manifest_path = trace_root / "manifest.json" + trace_root = safe_trace_path(trace_root, ".") + manifest_path = safe_trace_path(trace_root, "manifest.json") try: manifest = json.loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: @@ -315,9 +833,9 @@ def bind_prompt_provenance(trace_root: Path, provenance: dict[str, object]) -> N if not isinstance(record, dict) or not isinstance(data, bytes): raise PreflightError("validated prompt provenance is invalid") digest = sha256_bytes(data) - provenance_root = trace_root / "provenance" + provenance_root = safe_trace_path(trace_root, "provenance") provenance_root.mkdir(parents=True, exist_ok=True) - destination = provenance_root / f"{digest}.json" + destination = safe_trace_path(trace_root, Path("provenance") / f"{digest}.json") if destination.exists() and destination.read_bytes() != data: raise PreflightError(f"content-addressed provenance collision: {destination}") if not destination.exists(): diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index d41c87c4b9bd..7db2c0f71bf9 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -14,12 +14,15 @@ bind_prompt_provenance, resolved, run_preflight, + seal_audits, validate_prompt_provenance, + verify_sealed_audits, write_audits, ) -from trace_format import CORPUS_SHA256, MODEL_SHA256, TraceBundle, TraceError, sha256_file +from trace_format import ADMITTED_UBATCH, CORPUS_SHA256, MODEL_SHA256, TraceBundle, TraceError, sha256_file DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" +APPROVED_EXPORTERS: dict[str, str] = {} def git_output(checkout: Path, *args: str) -> str: @@ -41,6 +44,13 @@ def verify_checkout(checkout: Path) -> str: return revision +def verify_exporter_approval(exporter_sha256: str) -> None: + if APPROVED_EXPORTERS.get(exporter_sha256) != DS4_REVISION: + raise PreflightError( + "ds4 trace exporter is not approved for the pinned ds4 revision; " + "publish and review the exporter before cross-runtime execution") + + def preflight(args: argparse.Namespace) -> dict[str, object]: checkout = resolved(args.checkout) revision = verify_checkout(checkout) @@ -50,7 +60,7 @@ def preflight(args: argparse.Namespace) -> dict[str, object]: model=args.model, prompt=args.prompt, output=args.output, - watchdog_pid_file=args.watchdog_pid_file, + repo=args.repo, busy_patterns=args.busy_pattern, ) result.update({ @@ -69,10 +79,10 @@ def preflight(args: argparse.Namespace) -> dict[str, object]: def main() -> int: parser = argparse.ArgumentParser(description="Fail-closed launcher for the pinned ds4 trace exporter") parser.add_argument("--checkout", type=Path, default=Path("/home/papa/src/ds4-v41")) + parser.add_argument("--repo", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--prompt", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--watchdog-pid-file", type=Path, required=True) parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) parser.add_argument("--exporter", type=Path, required=True) parser.add_argument("--exporter-sha256", required=True) @@ -81,11 +91,15 @@ def main() -> int: parser.add_argument("--prompt-provenance", type=Path, required=True) parser.add_argument("--context", type=int, default=32768) parser.add_argument("--decode-steps", type=int, default=8) - parser.add_argument("--prefill-chunk", type=int, default=512) + parser.add_argument("--prefill-chunk", type=int, default=ADMITTED_UBATCH) parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() try: + if args.prefill_chunk != ADMITTED_UBATCH: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require admitted prefill chunk {ADMITTED_UBATCH}, " + f"found {args.prefill_chunk}") if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") if args.preflight_only: @@ -100,6 +114,7 @@ def main() -> int: if exporter_sha256 != args.exporter_sha256: raise PreflightError( f"trace exporter SHA-256 mismatch: expected {args.exporter_sha256}, found {exporter_sha256}") + verify_exporter_approval(exporter_sha256) model_sha256 = sha256_file(resolved(args.model)) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") @@ -117,6 +132,7 @@ def main() -> int: preflight_audit = preflight(args) preflight_audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) + pre_audit_digests = seal_audits(pre_audits) command = [ str(exporter), "--model", str(resolved(args.model)), @@ -133,6 +149,7 @@ def main() -> int: result = subprocess.run(command, cwd=resolved(args.checkout), check=False) if result.returncode != 0: return result.returncode + verify_sealed_audits(pre_audits, pre_audit_digests) postflight_audit = preflight(args) post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 522ab868b8aa..f313a4a58607 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -15,10 +15,25 @@ bind_prompt_provenance, resolved, run_preflight, + safe_trace_path, + seal_audits, validate_prompt_provenance, + verify_sealed_audits, write_audits, ) -from trace_format import CORPUS_SHA256, MODEL_SHA256, REPOSITORY, TraceBundle, TraceError, sha256_file +from trace_format import ( + ADMITTED_BATCH, + ADMITTED_UBATCH, + CORPUS_SHA256, + MODEL_SHA256, + REPOSITORY, + REQUIRED_EXPERT_CACHE_BYTES, + REQUIRED_EXPERT_CACHE_MIB, + REQUIRED_EXPERT_SLOTS, + TraceBundle, + TraceError, + sha256_file, +) def git_output(repo: Path, *args: str) -> bytes: @@ -76,8 +91,10 @@ def candidate_attestation(args: argparse.Namespace, exporter_sha256: str) -> dic } -def bind_candidate_attestation(output: Path, attestation: dict[str, str]) -> None: - manifest_path = output / "manifest.json" +def bind_candidate_attestation( + output: Path, + attestation: dict[str, str]) -> None: + manifest_path = safe_trace_path(output, "manifest.json") try: manifest = json.loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: @@ -98,15 +115,38 @@ def build_command(args: argparse.Namespace, exporter: Path, output: Path) -> lis "-n", str(args.decode_steps), "-b", str(args.batch), "-ub", str(args.ubatch), + "--device", args.device, "-ngl", str(args.gpu_layers), "-fa", "on", "-ctk", "f16", "-ctv", "f16", + "--load-mode", "none", "--expert-cache-slots", str(args.expert_cache_slots), "--expert-cache-mib", str(args.expert_cache_mib), ] +def validate_runtime_config(args: argparse.Namespace) -> None: + if args.batch != ADMITTED_BATCH: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require batch {ADMITTED_BATCH}, found {args.batch}") + if args.ubatch != ADMITTED_UBATCH: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require admitted ubatch {ADMITTED_UBATCH}, found {args.ubatch}") + if args.expert_cache_slots != REQUIRED_EXPERT_SLOTS: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require {REQUIRED_EXPERT_SLOTS} expert cache slots, " + f"found {args.expert_cache_slots}") + if args.expert_cache_mib != REQUIRED_EXPERT_CACHE_MIB: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require {REQUIRED_EXPERT_CACHE_BYTES} expert cache bytes " + f"({REQUIRED_EXPERT_CACHE_MIB} MiB), found {args.expert_cache_mib} MiB") + if args.device != "ROCm0": + raise PreflightError(f"DeepSeek V4.1 correctness runs require device ROCm0, found {args.device}") + if args.gpu_layers != 99: + raise PreflightError(f"DeepSeek V4.1 correctness runs require 99 GPU layers, found {args.gpu_layers}") + + def main() -> int: parser = argparse.ArgumentParser(description="Fail-closed launcher for llama.cpp DeepSeek V4.1 traces") parser.add_argument("--exporter", type=Path, required=True) @@ -120,19 +160,20 @@ def main() -> int: parser.add_argument("--model", type=Path, required=True) parser.add_argument("--prompt", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--watchdog-pid-file", type=Path, required=True) parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) parser.add_argument("--context", type=int, default=32768) parser.add_argument("--decode-steps", type=int, default=8) - parser.add_argument("--batch", type=int, default=2048) - parser.add_argument("--ubatch", type=int, default=512) - parser.add_argument("--expert-cache-slots", type=int, required=True) - parser.add_argument("--expert-cache-mib", type=int, required=True) + parser.add_argument("--batch", type=int, default=ADMITTED_BATCH) + parser.add_argument("--ubatch", type=int, default=ADMITTED_UBATCH) + parser.add_argument("--device", default="ROCm0") + parser.add_argument("--expert-cache-slots", type=int, default=REQUIRED_EXPERT_SLOTS) + parser.add_argument("--expert-cache-mib", type=int, default=REQUIRED_EXPERT_CACHE_MIB) parser.add_argument("--gpu-layers", type=int, default=99) parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() try: + validate_runtime_config(args) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") if args.preflight_only: @@ -140,7 +181,7 @@ def main() -> int: model=args.model, prompt=args.prompt, output=args.output, - watchdog_pid_file=args.watchdog_pid_file, + repo=args.repo, busy_patterns=args.busy_pattern, ) print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) @@ -169,7 +210,7 @@ def main() -> int: model=args.model, prompt=args.prompt, output=args.output, - watchdog_pid_file=args.watchdog_pid_file, + repo=args.repo, busy_patterns=args.busy_pattern, ) preflight_audit["runtime"] = "llama.cpp" @@ -178,11 +219,13 @@ def main() -> int: "decode_steps": args.decode_steps, "batch": args.batch, "ubatch": args.ubatch, + "device": args.device, "expert_cache_slots": args.expert_cache_slots, "expert_cache_mib": args.expert_cache_mib, "gpu_layers": args.gpu_layers, } pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) + pre_audit_digests = seal_audits(pre_audits) environment = os.environ.copy() environment["DSV41_TRACE_MEMORY_AUDIT"] = pre_audits["memory"] environment["DSV41_TRACE_SWAP_AUDIT"] = pre_audits["swap"] @@ -192,11 +235,12 @@ def main() -> int: result = subprocess.run(command, env=environment, check=False) if result.returncode != 0: return result.returncode + verify_sealed_audits(pre_audits, pre_audit_digests) postflight_audit = run_preflight( model=args.model, prompt=args.prompt, output=args.output, - watchdog_pid_file=args.watchdog_pid_file, + repo=args.repo, busy_patterns=args.busy_pattern, ) postflight_audit["runtime"] = "llama.cpp" diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index f8209d1337a9..83ec25d9cdd3 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -9,7 +9,17 @@ from pathlib import Path from preflight import PreflightError, require_nvme_path, resolved, run_preflight -from trace_format import CORPUS_SHA256, MODEL_SHA256, TraceBundle, report, sha256_file +from trace_format import ( + ADMITTED_BATCH, + ADMITTED_UBATCH, + CORPUS_SHA256, + MODEL_SHA256, + REQUIRED_EXPERT_CACHE_MIB, + REQUIRED_EXPERT_SLOTS, + TraceBundle, + report, + sha256_file, +) CORPORA = ( "correctness-prose.txt", @@ -81,27 +91,46 @@ def main() -> int: parser.add_argument("--repo", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--watchdog-pid-file", type=Path, required=True) parser.add_argument("--llama-runner", type=Path, required=True) parser.add_argument("--llama-exporter", type=Path, required=True) parser.add_argument("--llama-prompt-builder", type=Path, required=True) parser.add_argument("--candidate-revision", required=True) parser.add_argument("--base-revision", required=True) parser.add_argument("--candidate-diff-sha256", required=True) - parser.add_argument("--ds4-runner", type=Path, required=True) - parser.add_argument("--ds4-exporter", type=Path, required=True) - parser.add_argument("--ds4-exporter-sha256", required=True) + parser.add_argument("--ds4-runner", type=Path) + parser.add_argument("--ds4-exporter", type=Path) + parser.add_argument("--ds4-exporter-sha256") parser.add_argument("--ds4-checkout", type=Path, default=Path("/home/papa/src/ds4-v41")) + parser.add_argument("--llama-only", action="store_true") parser.add_argument("--contexts", type=int, nargs="+", default=[32768]) - parser.add_argument("--ubatches", type=int, nargs="+", default=[512]) + parser.add_argument("--ubatches", type=int, nargs="+", default=[ADMITTED_UBATCH]) parser.add_argument("--decode-steps", type=int, default=8) - parser.add_argument("--batch", type=int, default=2048) - parser.add_argument("--expert-cache-slots", type=int, required=True) - parser.add_argument("--expert-cache-mib", type=int, required=True) + parser.add_argument("--batch", type=int, default=ADMITTED_BATCH) + parser.add_argument("--device", default="ROCm0") + parser.add_argument("--expert-cache-slots", type=int, default=REQUIRED_EXPERT_SLOTS) + parser.add_argument("--expert-cache-mib", type=int, default=REQUIRED_EXPERT_CACHE_MIB) parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) args = parser.parse_args() try: + if args.ubatches != [ADMITTED_UBATCH]: + raise PreflightError( + f"DeepSeek V4.1 correctness matrix requires admitted ubatch [{ADMITTED_UBATCH}]") + if args.batch != ADMITTED_BATCH: + raise PreflightError(f"DeepSeek V4.1 correctness matrix requires batch {ADMITTED_BATCH}") + if args.device != "ROCm0": + raise PreflightError("DeepSeek V4.1 correctness matrix requires device ROCm0") + if args.expert_cache_slots != REQUIRED_EXPERT_SLOTS: + raise PreflightError( + f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_SLOTS} expert cache slots") + if args.expert_cache_mib != REQUIRED_EXPERT_CACHE_MIB: + raise PreflightError( + f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_CACHE_MIB} MiB expert cache") + if not args.llama_only and ( + args.ds4_runner is None or args.ds4_exporter is None or args.ds4_exporter_sha256 is None): + raise PreflightError( + "--ds4-runner, --ds4-exporter, and --ds4-exporter-sha256 are required " + "unless --llama-only is selected") repo = resolved(args.repo) output = require_nvme_path(args.output, "matrix output") model = require_nvme_path(args.model, "model") @@ -110,6 +139,17 @@ def main() -> int: model_sha256 = sha256_file(model) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + initial_corpus = require_nvme_path( + repo / "tests" / "corpus" / CORPORA[0], + "repository corpus", + ) + run_preflight( + model=model, + prompt=initial_corpus, + output=output, + repo=repo, + busy_patterns=args.busy_pattern, + ) prompt_builder = resolved(args.llama_prompt_builder) if not prompt_builder.is_file() or not os.access(prompt_builder, os.X_OK): raise PreflightError(f"prompt builder is not executable: {prompt_builder}") @@ -155,7 +195,7 @@ def main() -> int: model=model, prompt=Path(corpus["path"]), output=prompt, - watchdog_pid_file=args.watchdog_pid_file, + repo=repo, busy_patterns=args.busy_pattern, ) prepared = prepare_prompt( @@ -184,22 +224,26 @@ def main() -> int: "--prompt-provenance", provenance, "--corpus-name", corpus["name"], "--corpus-sha256", corpus["sha256"], - "--watchdog-pid-file", str(resolved(args.watchdog_pid_file)), "--context", str(context), "--decode-steps", str(args.decode_steps), ] for pattern in args.busy_pattern: common.extend(["--busy-pattern", pattern]) - run([ - sys.executable, - str(resolved(args.ds4_runner)), - "--checkout", str(resolved(args.ds4_checkout)), - "--exporter", str(resolved(args.ds4_exporter)), - "--exporter-sha256", args.ds4_exporter_sha256, - "--output", str(ds4_output), - "--prefill-chunk", str(ubatch), - *common, - ]) + if not args.llama_only: + assert args.ds4_runner is not None + assert args.ds4_exporter is not None + assert args.ds4_exporter_sha256 is not None + run([ + sys.executable, + str(resolved(args.ds4_runner)), + "--repo", str(repo), + "--checkout", str(resolved(args.ds4_checkout)), + "--exporter", str(resolved(args.ds4_exporter)), + "--exporter-sha256", args.ds4_exporter_sha256, + "--output", str(ds4_output), + "--prefill-chunk", str(ubatch), + *common, + ]) run([ sys.executable, str(resolved(args.llama_runner)), @@ -211,23 +255,35 @@ def main() -> int: "--output", str(llama_output), "--batch", str(args.batch), "--ubatch", str(ubatch), + "--device", args.device, "--expert-cache-slots", str(args.expert_cache_slots), "--expert-cache-mib", str(args.expert_cache_mib), *common, ]) - comparison = report(TraceBundle(ds4_output), TraceBundle(llama_output)) - result_path = output / "reports" / f"{case}.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text( - json.dumps(comparison, sort_keys=True, separators=(",", ":")) + "\n", - encoding="ascii", - ) - results.append({"case": case, **comparison}) - if comparison["status"] != "TARGET PASS": - raise RuntimeError(f"correctness mismatch in {case}: {comparison['first_divergence']}") + if args.llama_only: + results.append({ + "case": case, + "status": "BRINGUP TRACE CAPTURED", + "cross_runtime_status": "INCOMPLETE", + "trace": str(llama_output), + }) + else: + comparison = report(TraceBundle(ds4_output), TraceBundle(llama_output)) + result_path = output / "reports" / f"{case}.json" + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text( + json.dumps(comparison, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + results.append({"case": case, **comparison}) + if comparison["status"] != "TARGET PASS": + raise RuntimeError( + f"correctness mismatch in {case}: {comparison['first_divergence']}") summary = { - "status": "TARGET PASS", + "status": "BRINGUP TRACE CAPTURED" if args.llama_only else "TARGET PASS", + "mode": "llama-only" if args.llama_only else "cross-runtime", + "cross_runtime_status": "INCOMPLETE" if args.llama_only else "TARGET PASS", "model": str(model), "model_sha256": model_sha256, "candidate_revision": args.candidate_revision, diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 594e000a4ded..b8fe07ac4c63 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -17,6 +17,23 @@ MODEL_SHA256 = "1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42" REPOSITORY = "halo-box/strix-llama.cpp" SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 +WATCHDOG_EMERGENCY_LIMIT = 118 * 1024 * 1024 * 1024 +STRICT_MEMORY_LIMIT = 120 * 1024 * 1024 * 1024 +WATCHDOG_LEASE_FORMAT = "strix-memory-watchdog-lease" +WATCHDOG_VERSION = 2 +WATCHDOG_REVISION = "778db6f50eae04e6c232c69b9575bdbd0747962b" +WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f" +APPROVED_WATCHDOGS = {WATCHDOG_SCRIPT_SHA256: WATCHDOG_REVISION} +ADMITTED_UBATCH = 32 +ADMITTED_BATCH = 2048 +EXPERT_COUNT = 384 +EXPERTS_USED = 6 +EXPERT_SLOT_BYTES = 398_131_200 +REQUIRED_EXPERT_SLOTS = min(EXPERT_COUNT, EXPERTS_USED * ADMITTED_UBATCH) +REQUIRED_EXPERT_CACHE_BYTES = REQUIRED_EXPERT_SLOTS * EXPERT_SLOT_BYTES +REQUIRED_EXPERT_CACHE_MIB = REQUIRED_EXPERT_CACHE_BYTES // (1024 * 1024) +RAW_ATTENTION_LAYERS = (0, 1) +RAW_ATTENTION_WIDTH = 128 CORPUS_SHA256 = { "correctness-prose.txt": "2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2", "correctness-code.txt": "41b4246ef4e6b4e3f9f23a3d02aa8cdab48f495b3af0ebeaccea255679c771f0", @@ -329,9 +346,11 @@ def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: class TraceBundle: def __init__(self, root: Path, verify_blobs: bool = True): - self.root = root + if root.is_symlink(): + raise TraceError("trace root must not be a symlink") + self.root = root.resolve() try: - self.manifest = json.loads((root / MANIFEST_NAME).read_text(encoding="ascii")) + self.manifest = json.loads(self._path(MANIFEST_NAME).read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise TraceError(f"cannot read manifest: {error}") from error if self.manifest.get("trace_format") != TRACE_FORMAT: @@ -348,7 +367,7 @@ def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: result = [] try: stream: BinaryIO - with (self.root / EVENTS_NAME).open("rb") as stream: + with self._path(EVENTS_NAME).open("rb") as stream: for line_number, raw in enumerate(stream, 1): if not raw.endswith(b"\n"): raise TraceError(f"events.jsonl is truncated at line {line_number}") @@ -420,7 +439,7 @@ def _validate_manifest(self) -> None: if provenance.get("path") != f"provenance/{provenance_sha256}.json": raise TraceError("prompt provenance path is not content addressed") try: - provenance_bytes = (self.root / provenance["path"]).read_bytes() + provenance_bytes = self._path(provenance["path"]).read_bytes() provenance_record = json.loads(provenance_bytes.decode("ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise TraceError(f"cannot read prompt provenance: {error}") from error @@ -471,12 +490,28 @@ def _validate_manifest(self) -> None: "candidate_topk_blocks": 2048, "candidate_block_size": 8, "index_top_k": 512, + "raw_attention_layers": list(RAW_ATTENTION_LAYERS), + "raw_attention_width": RAW_ATTENTION_WIDTH, "candidate_propagation_layers": [24, 28, 32, 36], } if self.manifest["config"].get("deepseek41") != expected_config: raise TraceError("DeepSeek V4.1 configuration is invalid") if self.manifest["comparison"].get("logits") != "byte-identical-f32": raise TraceError("logit comparison policy must be byte-identical-f32") + config = self.manifest["config"] + if self.manifest["runtime"] == "llama.cpp": + if config.get("batch") != ADMITTED_BATCH or config.get("ubatch") != ADMITTED_UBATCH: + raise TraceError("llama.cpp trace does not use the admitted batch and ubatch") + if config.get("expert_cache_slots") != REQUIRED_EXPERT_SLOTS or ( + config.get("expert_cache_bytes") != REQUIRED_EXPERT_CACHE_BYTES): + raise TraceError("llama.cpp trace does not use the admitted expert cache") + if config.get("device") != "ROCm0" or config.get("gpu_layers") != 99: + raise TraceError("llama.cpp trace does not use the required ROCm0 offload") + if config.get("kv_type_k") != "f16" or config.get("kv_type_v") != "f16" or ( + config.get("flash_attention") not in (True, 1)) or config.get("load_mode") != 0: + raise TraceError("llama.cpp trace inference configuration is invalid") + if self.manifest["runtime"] == "ds4" and config.get("prefill_chunk") != ADMITTED_UBATCH: + raise TraceError("ds4 trace does not use the admitted prefill chunk") for audit_phase in ("pre", "post"): phase_audits = self.manifest["audits"].get(audit_phase) if not isinstance(phase_audits, dict): @@ -498,7 +533,7 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: expected_path = f"audits/{phase}/{digest}.json" if audit_path != expected_path: raise TraceError(f"manifest {phase} {kind} audit path is not content addressed") - evidence_path = self.root / audit_path + evidence_path = self._path(audit_path) try: evidence = evidence_path.read_bytes() except OSError as error: @@ -511,6 +546,8 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} {kind} audit evidence is invalid: {error}") from error if record.get("kind") != kind or record.get("created_unix") != audit["created_unix"]: raise TraceError(f"{phase} {kind} audit evidence metadata mismatch") + if record.get("environment") != {"HIP_LAUNCH_BLOCKING": "1"}: + raise TraceError(f"{phase} {kind} audit environment is invalid") if not isinstance(record.get("data"), dict): raise TraceError(f"{phase} {kind} audit evidence data is invalid") if kind == "memory": @@ -521,34 +558,147 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: if record["data"].get("enabled") is not False or record["data"].get("entries") != []: raise TraceError(f"{phase} swap audit evidence does not report zero configured swap") if kind == "watchdog": - required = ("pid", "start_time_ticks", "command_sha256", "heartbeat_path", "heartbeat_unix") + required = ( + "format", + "version", + "lease_id", + "lease_path", + "watchdog_pid", + "watchdog_start_time_ticks", + "watchdog_command", + "watchdog_command_sha256", + "watchdog_executable_path", + "watchdog_script_path", + "watchdog_script_sha256", + "watchdog_revision", + "soft_bytes", + "emergency_bytes", + "strict_ceiling_bytes", + "grace_seconds", + "sample_interval_seconds", + "procfs_root", + "guardian_pid", + "child_pid", + "child_process_group_id", + "command", + "child_command_sha256", + "heartbeat_path", + "heartbeat_unix", + "max_heartbeat_age_seconds", + "audit_live_path", + "audit_device", + "audit_inode", + "audit_uid", + "audit_mode", + "audit_fd", + "audit", + ) if any(key not in record["data"] for key in required): raise TraceError(f"{phase} watchdog audit evidence is incomplete") data = record["data"] - if not isinstance(data["pid"], int) or data["pid"] <= 1: + if data["format"] != WATCHDOG_LEASE_FORMAT or data["version"] != WATCHDOG_VERSION: + raise TraceError(f"{phase} watchdog audit format is invalid") + if not isinstance(data["lease_id"], str) or re.fullmatch(r"[0-9a-f]{32,64}", data["lease_id"]) is None: + raise TraceError(f"{phase} watchdog audit lease ID is invalid") + if not isinstance(data["watchdog_pid"], int) or data["watchdog_pid"] <= 1: raise TraceError(f"{phase} watchdog audit PID is invalid") - if not isinstance(data["start_time_ticks"], int) or data["start_time_ticks"] <= 0: + if not isinstance(data["watchdog_start_time_ticks"], int) or data["watchdog_start_time_ticks"] <= 0: raise TraceError(f"{phase} watchdog audit start time is invalid") - if not isinstance(data["command_sha256"], str) or re.fullmatch( - r"[0-9a-f]{64}", data["command_sha256"]) is None: + for key in ("watchdog_command_sha256", "watchdog_script_sha256"): + if not isinstance(data[key], str) or re.fullmatch(r"[0-9a-f]{64}", data[key]) is None: + raise TraceError(f"{phase} watchdog audit {key} is invalid") + for key in ("lease_path", "watchdog_command", "watchdog_script_path", "heartbeat_path", "audit_live_path"): + if not isinstance(data[key], str) or not data[key]: + raise TraceError(f"{phase} watchdog audit {key} is invalid") + if APPROVED_WATCHDOGS.get(data["watchdog_script_sha256"]) != data["watchdog_revision"]: + raise TraceError(f"{phase} watchdog audit revision is invalid") + if not isinstance(data["watchdog_executable_path"], str) or not data["watchdog_executable_path"]: + raise TraceError(f"{phase} watchdog executable path is invalid") + if data["soft_bytes"] != SOFT_MEMORY_LIMIT or ( + data["emergency_bytes"] != WATCHDOG_EMERGENCY_LIMIT) or ( + data["strict_ceiling_bytes"] != STRICT_MEMORY_LIMIT): + raise TraceError(f"{phase} watchdog audit thresholds are invalid") + if data["grace_seconds"] != 30.0 or data["sample_interval_seconds"] != 1.0: + raise TraceError(f"{phase} watchdog timing policy is invalid") + if data["procfs_root"] != "/proc": + raise TraceError(f"{phase} watchdog procfs root is invalid") + if not isinstance(data["guardian_pid"], int) or data["guardian_pid"] <= 1 or ( + not isinstance(data["child_pid"], int) or data["child_pid"] <= 1) or ( + not isinstance(data["child_process_group_id"], int) or data["child_process_group_id"] <= 1): + raise TraceError(f"{phase} watchdog child identity is invalid") + for key in ("audit_device", "audit_inode", "audit_uid", "audit_fd"): + if not isinstance(data[key], int) or data[key] < 0: + raise TraceError(f"{phase} watchdog audit {key} is invalid") + if data["audit_mode"] != 0o600: + raise TraceError(f"{phase} watchdog audit mode is invalid") + if not isinstance(data["command"], list) or not data["command"] or ( + not all(isinstance(argument, str) for argument in data["command"])): + raise TraceError(f"{phase} watchdog child command is invalid") + canonical_command = json.dumps( + data["command"], ensure_ascii=True, separators=(",", ":")).encode("utf-8") + if data["child_command_sha256"] != sha256_bytes(canonical_command): + raise TraceError(f"{phase} watchdog child command SHA-256 is invalid") + if not isinstance(data["watchdog_command_sha256"], str) or re.fullmatch( + r"[0-9a-f]{64}", data["watchdog_command_sha256"]) is None: raise TraceError(f"{phase} watchdog audit command SHA-256 is invalid") - if not isinstance(data["heartbeat_path"], str) or not data["heartbeat_path"]: - raise TraceError(f"{phase} watchdog audit heartbeat path is invalid") if not isinstance(data["heartbeat_unix"], int) or data["heartbeat_unix"] <= 0: raise TraceError(f"{phase} watchdog audit heartbeat timestamp is invalid") max_age = data.get("max_heartbeat_age_seconds") - if not isinstance(max_age, int) or max_age <= 0 or max_age > 30: + if not isinstance(max_age, (int, float)) or isinstance(max_age, bool) or ( + max_age <= 0 or max_age > 30): raise TraceError(f"{phase} watchdog audit heartbeat age is invalid") if data["heartbeat_unix"] > record["created_unix"] or ( record["created_unix"] - data["heartbeat_unix"] > max_age): raise TraceError(f"{phase} watchdog audit heartbeat was stale when captured") + audit_jsonl = data["audit"] + if not isinstance(audit_jsonl, dict): + raise TraceError(f"{phase} watchdog JSONL reference is invalid") + jsonl_digest = audit_jsonl.get("sha256", "") + if not isinstance(jsonl_digest, str) or re.fullmatch(r"[0-9a-f]{64}", jsonl_digest) is None: + raise TraceError(f"{phase} watchdog JSONL SHA-256 is invalid") + if audit_jsonl.get("path") != f"audits/{phase}/{jsonl_digest}.jsonl": + raise TraceError(f"{phase} watchdog JSONL path is invalid") + if not isinstance(audit_jsonl.get("event_count"), int) or audit_jsonl["event_count"] < 2: + raise TraceError(f"{phase} watchdog JSONL event count is invalid") + jsonl_path = self._path(audit_jsonl["path"]) + try: + jsonl_bytes = jsonl_path.read_bytes() + except OSError as error: + raise TraceError(f"cannot read {phase} watchdog JSONL audit: {error}") from error + if sha256_bytes(jsonl_bytes) != jsonl_digest: + raise TraceError(f"{phase} watchdog JSONL SHA-256 mismatch") + lines = jsonl_bytes.decode("ascii").splitlines() + if len(lines) != audit_jsonl["event_count"]: + raise TraceError(f"{phase} watchdog JSONL event count mismatch") + try: + events = [json.loads(line) for line in lines] + except json.JSONDecodeError as error: + raise TraceError(f"{phase} watchdog JSONL is invalid: {error}") from error + if not any(event.get("event") == "preflight" for event in events) or ( + not any(event.get("event") == "child_started" for event in events)): + raise TraceError(f"{phase} watchdog JSONL lacks startup evidence") def read_blob(self, event: dict[str, Any]) -> bytes: try: - return (self.root / event["blob"]).read_bytes() + return self._path(event["blob"]).read_bytes() except OSError as error: raise TraceError(f"cannot read blob {event['blob']}: {error}") from error + def _path(self, relative: str) -> Path: + relative_path = Path(relative) + if relative_path.is_absolute() or ".." in relative_path.parts: + raise TraceError(f"trace path is outside the bundle: {relative}") + candidate = self.root + for part in relative_path.parts: + candidate = candidate / part + if candidate.is_symlink(): + raise TraceError(f"trace path must not use symlinks: {relative}") + try: + candidate.resolve().relative_to(self.root) + except ValueError as error: + raise TraceError(f"trace path is outside the bundle: {relative}") from error + return candidate + def _validate_coverage(self) -> None: expected = self.manifest.get("expected") if not isinstance(expected, dict): @@ -566,6 +716,24 @@ def _validate_coverage(self) -> None: raise TraceError("expected prompt_tokens does not match prompt provenance") for event in self.events: self._validate_component_schema(event) + raw_attention = { + (event["phase"], event["step"], event["token_start"], event["layer"]): event + for event in self.events + if event["component"] == "attn.source" and event["layer"] in RAW_ATTENTION_LAYERS + } + raw_coordinates = { + (phase, step, token_start) + for phase, step, token_start, _layer in raw_attention + } + for phase, step, token_start in raw_coordinates: + layer0 = raw_attention.get((phase, step, token_start, 0)) + layer1 = raw_attention.get((phase, step, token_start, 1)) + if layer0 is None or layer1 is None: + continue + if layer0["shape"] != layer1["shape"] or self.read_blob(layer0) != self.read_blob(layer1): + raise TraceError( + f"raw attn.source differs between layers 0 and 1 at " + f"{phase} step {step} token {token_start}") if not isinstance(components, dict): raise TraceError("expected components are invalid") if self.manifest.get("model", {}).get("architecture") == "deepseek41": @@ -678,9 +846,26 @@ def _validate_component_schema(self, event: dict[str, Any]) -> None: raise TraceError(f"{component} dtype must be {expected_dtype}") if width is not None and shape[0] != width: raise TraceError(f"{component} shape must be [{width},token_count]") + if component == "attn.source" and layer in RAW_ATTENTION_LAYERS: + if shape[0] != RAW_ATTENTION_WIDTH: + raise TraceError(f"raw attn.source shape must be [{RAW_ATTENTION_WIDTH},token_count]") + values = list(struct.iter_unpack(" config.get("candidate_topk_blocks", 0): raise TraceError("attn.candidate_blocks width exceeds candidate_topk_blocks") - if component in ("attn.source", "attn.candidates") and shape[0] > config.get("index_top_k", 0): + if component == "attn.source" and layer not in RAW_ATTENTION_LAYERS and ( + shape[0] > config.get("index_top_k", 0)): + raise TraceError(f"{component} width exceeds index_top_k") + if component == "attn.candidates" and shape[0] > config.get("index_top_k", 0): raise TraceError(f"{component} width exceeds index_top_k") if component == "expert.ids": values = struct.iter_unpack(" Mismatch | None: return None -def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: +def compare_bundles( + left: TraceBundle, + right: TraceBundle, + *, + runtime_roles: tuple[str, str] = ("ds4", "llama.cpp")) -> Mismatch | None: if left.root.resolve() == right.root.resolve(): return Mismatch( "artifact_identity", @@ -739,7 +928,7 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: None, "cannot compare a trace bundle with itself", ) - if left.manifest["runtime"] != "ds4" or right.manifest["runtime"] != "llama.cpp": + if left.manifest["runtime"] != runtime_roles[0] or right.manifest["runtime"] != runtime_roles[1]: return Mismatch( "runtime_role", "manifest", @@ -747,7 +936,7 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: -1, -1, None, - "left trace must be pinned ds4 and right trace must be llama.cpp candidate", + f"left trace must be {runtime_roles[0]} and right trace must be {runtime_roles[1]}", ) mismatch = compare_manifests(left, right) if mismatch is not None: @@ -864,11 +1053,16 @@ def compare_bundles(left: TraceBundle, right: TraceBundle) -> Mismatch | None: )) -def report(left: TraceBundle, right: TraceBundle) -> dict[str, Any]: - mismatch = compare_bundles(left, right) +def report( + left: TraceBundle, + right: TraceBundle, + *, + runtime_roles: tuple[str, str] = ("ds4", "llama.cpp"), + success_status: str = "TARGET PASS") -> dict[str, Any]: + mismatch = compare_bundles(left, right, runtime_roles=runtime_roles) if mismatch is None: return { - "status": "TARGET PASS", + "status": success_status, "trace_version": TRACE_VERSION, "left_runtime": left.manifest.get("runtime"), "right_runtime": right.manifest.get("runtime"), @@ -904,6 +1098,61 @@ def command_compare(args: argparse.Namespace) -> int: return 0 if result["status"] == "TARGET PASS" else 1 +def local_report(left: TraceBundle, right: TraceBundle, mode: str) -> dict[str, Any]: + result = report( + left, + right, + runtime_roles=("llama.cpp", "llama.cpp"), + success_status="BRINGUP PASS", + ) + result["mode"] = mode + result["cross_runtime_status"] = "INCOMPLETE" + result["cross_runtime_requirement"] = ( + "Run the pinned ds4 exporter and trace_format.py compare before reporting TARGET PASS.") + if mode == "self-consistency": + if left.manifest.get("candidate") != right.manifest.get("candidate"): + result = { + **result, + "status": "FAIL", + "first_divergence": Mismatch( + "candidate_identity", + "manifest", + "metadata", + -1, + -1, + None, + "self-consistency traces use different candidate attestations", + ).as_dict(), + } + else: + base_revision = left.manifest.get("candidate", {}).get("revision") + integrated_base = right.manifest.get("candidate", {}).get("base_revision") + if base_revision != integrated_base: + result = { + **result, + "status": "FAIL", + "first_divergence": Mismatch( + "candidate_identity", + "manifest", + "metadata", + -1, + -1, + None, + f"base trace revision {base_revision!r} != integrated oracle {integrated_base!r}", + ).as_dict(), + } + return result + + +def command_compare_local(args: argparse.Namespace) -> int: + result = local_report(TraceBundle(args.left), TraceBundle(args.right), args.mode) + text = canonical_json(result) + "\n" + if args.report: + args.report.write_text(text, encoding="ascii") + sys.stdout.write(text) + return 0 if result["status"] == "BRINGUP PASS" else 1 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Validate and compare DeepSeek V4.1 correctness traces") subparsers = parser.add_subparsers(dest="command", required=True) @@ -915,6 +1164,12 @@ def build_parser() -> argparse.ArgumentParser: compare_parser.add_argument("right", type=Path) compare_parser.add_argument("--report", type=Path) compare_parser.set_defaults(func=command_compare) + local_parser = subparsers.add_parser("compare-local") + local_parser.add_argument("mode", choices=("self-consistency", "base-regression")) + local_parser.add_argument("left", type=Path) + local_parser.add_argument("right", type=Path) + local_parser.add_argument("--report", type=Path) + local_parser.set_defaults(func=command_compare_local) return parser diff --git a/tools/deepseek-v41-trace/verify_ds4_anchors.py b/tools/deepseek-v41-trace/verify_ds4_anchors.py new file mode 100644 index 000000000000..5b8e552ae335 --- /dev/null +++ b/tools/deepseek-v41-trace/verify_ds4_anchors.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from trace_format import DS4_REVISION, sha256_file + + +ANCHORS = { + "tests/test-vectors/README.md": + "0e59b2f2832bed8af0a91e6ff20962debf964cd2d1d141c086e52cfcc995a1c3", + "tests/test-vectors/flash-0731/manifest.json": + "ebf237a5660a6851fb8085e77f532901a9d758208b25d7ed5af0b7af4b28f91b", + "tests/test-vectors/flash-0731/official.vec": + "77ae699889bfaf1348768dcbe7ea2c72279ae86abb10470d3e1b08cd1fd82a83", + "tests/test-vectors/flash-0731/local-golden.vec": + "23d942ff3b9bb2a3f82927d11aa3ed1461e1f302071e788d0d95a5c165e47d3b", + "tests/test_engram.c": + "198a561d981f62518a9d28035480a7e220b99c156cde6d248b8baabd684cc74b", + "ds4_engram.c": + "2b6ca468510ebf45ee298a905525bc7234dacad9a384bf2011eba19ba2c0bdf7", + "ds4_engram.h": + "f84a264e0fe199d23a6f0c56fbbd19e222adc7009eed13c185af6403f68f7f5c", + "tests/test_deepseek41_metal.c": + "9197c2f9d65b380ce25be5334991e4bfaf40e82e6708e64f2b9329552412d28c", + "tests/test_deepseek41_graph.c": + "6dc786f831c93ae7f5aa56e7518f67657125f7c0fd35cead3c646eff3d3f9e09", + "tests/test_deepseek41_prefill.c": + "452774b9332d393822d84288eeb2d71ca1fb25f1ba30d20a49160d1787de734f", + "tests/test_deepseek41_manifest.py": + "2d7aa1fc93805d9c97839c5de9eccc856628f13e6913c3bbc3f0c99b0827aeae", + "tests/test_deepseek41_conversion.py": + "a40b83062a9b91338773addd77fe62650296f4de607057cb4fd1329287f347b5c", + "tests/test_deepseek41_gguf.c": + "8f41e049d5ec179c38a1a00ac61712db0306902f0ef74bcdb989d8993316ff35", + "gguf-tools/deepseek41_metadata.py": + "39300bbd504165b97de017edd72563377f50b8a5511ca7478252a86f31a0009b", + "ds4.c": + "1776dbfed177ea14f3ce6cac1d8d0b1c1b44dfff2c2663769a9a5634aeec34e7", +} + + +class AnchorError(RuntimeError): + pass + + +def git_output(checkout: Path, *args: str) -> str: + try: + return subprocess.check_output( + ["git", "-C", str(checkout), *args], + text=True, + stderr=subprocess.STDOUT, + ).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise AnchorError(f"git {' '.join(args)} failed: {error}") from error + + +def verify( + checkout: Path, + *, + anchors: dict[str, str] = ANCHORS, + expected_revision: str = DS4_REVISION) -> dict[str, object]: + checkout = checkout.expanduser().resolve() + revision = git_output(checkout, "rev-parse", "HEAD") + if revision != expected_revision: + raise AnchorError(f"ds4 revision mismatch: expected {expected_revision}, found {revision}") + if git_output(checkout, "diff", "--name-only") or git_output(checkout, "diff", "--cached", "--name-only"): + raise AnchorError("ds4 tracked source differs from the pinned revision") + files = [] + for relative, expected in anchors.items(): + path = checkout / relative + if not path.is_file(): + raise AnchorError(f"pinned ds4 anchor is missing: {relative}") + actual = sha256_file(path) + if actual != expected: + raise AnchorError( + f"pinned ds4 anchor SHA-256 mismatch for {relative}: expected {expected}, found {actual}") + files.append({"path": relative, "sha256": actual}) + return { + "status": "ANCHORS VERIFIED", + "revision": revision, + "files": files, + "limitations": [ + "official.vec contains selected-token and top-logprob slices, not complete logits", + "local-golden.vec is a tolerant top-64 drift anchor", + "the Metal source fixture is not executable evidence on Strix", + "cross-runtime TARGET PASS still requires the pinned ds4 trace exporter", + ], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify pinned ds4 bring-up evidence anchors") + parser.add_argument( + "--checkout", + type=Path, + default=Path("/home/papa/src/ds4-v41"), + ) + args = parser.parse_args() + try: + print(json.dumps(verify(args.checkout), sort_keys=True, separators=(",", ":"))) + return 0 + except AnchorError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 0139bb297e09c2b9c97119819ac24679a5218719 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 01:17:39 -0700 Subject: [PATCH 20/56] deepseek41 : attest host architecture and storage Require the selected ROCm device to resolve through KFD as gfx1151 and prove every active path is backed by local non-rotational NVMe storage before execution. Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 220 ++++++++++++ tools/deepseek-v41-trace/CMakeLists.txt | 9 + tools/deepseek-v41-trace/README.md | 17 +- tools/deepseek-v41-trace/host-attestation.h | 327 ++++++++++++++++++ tools/deepseek-v41-trace/llama-trace.cpp | 72 +++- tools/deepseek-v41-trace/preflight.py | 155 ++++++++- tools/deepseek-v41-trace/prompt-builder.cpp | 20 +- tools/deepseek-v41-trace/run_llama.py | 73 +++- .../test-host-attestation.cpp | 183 ++++++++++ tools/deepseek-v41-trace/trace_format.py | 72 +++- 10 files changed, 1098 insertions(+), 50 deletions(-) create mode 100644 tools/deepseek-v41-trace/host-attestation.h create mode 100644 tools/deepseek-v41-trace/test-host-attestation.cpp diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index be301e1672d7..52bd2072706b 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -6,6 +6,7 @@ import sys import tempfile import unittest +from unittest import mock from argparse import Namespace from pathlib import Path @@ -46,12 +47,54 @@ ).encode("ascii") WATCHDOG_JSONL_SHA256 = trace.sha256_bytes(WATCHDOG_JSONL) +ACCELERATOR_ATTESTATION = { + "format": "dsv41-accelerator-attestation", + "version": 1, + "backend_device": "ROCm0", + "backend_description": "AMD Radeon Graphics", + "pci_device_id": "0000:c1:00.0", + "kfd_node": "1", + "gpu_id": 1234, + "gfx_target_version": 110501, + "architecture": "gfx1151", + "source": "linux-kfd-sysfs", +} + +def storage_record(path: str) -> dict[str, object]: + model_storage = path.startswith("/mnt/models") + mount_point = "/mnt/models" if model_storage else "/home" + source = "/dev/nvme1n1" if model_storage else "/dev/nvme0n1p3[/home]" + device_number = "259:0" if model_storage else "259:3" + nvme_device = "nvme1n1" if model_storage else "nvme0n1" + return { + "resolved_path": path, + "existing_path": path, + "mount_point": mount_point, + "filesystem_type": "xfs" if model_storage else "btrfs", + "mount_source": source, + "device_number": device_number, + "block_device_path": f"/sys/devices/pci/block/{nvme_device}", + "nvme_device": nvme_device, + "rotational": False, + } + + +STORAGE_ATTESTATION = { + "model": storage_record("/mnt/models/model.gguf"), + "prompt": storage_record("/home/prompt.txt"), + "output": storage_record("/home"), + "repository": storage_record("/home/repo"), + "temporary_directory": storage_record("/home/tmp"), +} + AUDIT_RECORDS = { "memory": { "created_unix": 1, "kind": "memory", "environment": {"HIP_LAUNCH_BLOCKING": "1"}, "data": {"mem_total_bytes": 128, "mem_available_bytes": 64, "mem_used_bytes": 64}, + "storage": STORAGE_ATTESTATION, + "accelerator": dict(ACCELERATOR_ATTESTATION), }, "swap": { "created_unix": 1, @@ -136,6 +179,7 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "revision": trace.DS4_REVISION if runtime == "ds4" else "a" * 40, "build": {"sha256": "3" * 64}, "model": {"sha256": trace.MODEL_SHA256, "byte_count": 123, "architecture": "deepseek41"}, + "accelerator": dict(ACCELERATOR_ATTESTATION), "prompt": { "sha256": trace.sha256_bytes(prompt), "byte_count": len(prompt), @@ -158,6 +202,8 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "expert_cache_slots": trace.REQUIRED_EXPERT_SLOTS, "expert_cache_bytes": trace.REQUIRED_EXPERT_CACHE_BYTES, "device": "ROCm0", + "device_architecture": "gfx1151", + "device_pci_id": "0000:c1:00.0", "gpu_layers": 99, "load_mode": 0, "deepseek41": { @@ -475,6 +521,13 @@ def replace_event_blob( class TraceFormatTests(unittest.TestCase): + def setUp(self) -> None: + self._require_nvme_path = preflight.require_nvme_path + preflight.require_nvme_path = lambda path, label, **kwargs: preflight.resolved(path) + + def tearDown(self) -> None: + preflight.require_nvme_path = self._require_nvme_path + def test_serialization_preserves_float_bits_and_hashes(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" @@ -610,6 +663,159 @@ def test_rejects_unadmitted_expert_cache_configuration(self) -> None: self.assertEqual(trace.REQUIRED_EXPERT_CACHE_BYTES, 76_441_190_400) self.assertEqual(trace.REQUIRED_EXPERT_CACHE_MIB, 72_900) + def test_accelerator_attestation_rejects_wrong_missing_and_spoofed_architecture(self) -> None: + valid = dict(ACCELERATOR_ATTESTATION) + self.assertEqual( + run_llama.validate_accelerator_attestation(valid), + valid, + ) + for key, value, message in ( + ("architecture", "gfx1100", "architecture mismatch"), + ("architecture", None, "architecture mismatch"), + ("backend_device", "ROCm1", "backend_device mismatch"), + ("gfx_target_version", 110500, "gfx_target_version mismatch"), + ("source", "environment", "source mismatch")): + invalid = dict(valid) + if value is None: + del invalid[key] + else: + invalid[key] = value + with self.assertRaisesRegex(preflight.PreflightError, message): + run_llama.validate_accelerator_attestation(invalid) + + spoofed = dict(valid) + spoofed["gfx_target_version"] = 110500 + spoofed["architecture"] = "gfx1151" + with self.assertRaisesRegex(preflight.PreflightError, "gfx_target_version mismatch"): + run_llama.validate_accelerator_attestation(spoofed) + + def test_accelerator_query_fails_closed(self) -> None: + valid_result = run_llama.subprocess.CompletedProcess( + ["exporter"], 0, json.dumps(ACCELERATOR_ATTESTATION), "") + with mock.patch.object(run_llama.subprocess, "run", return_value=valid_result): + self.assertEqual( + run_llama.query_accelerator_attestation(Path("/exporter"), "ROCm0"), + ACCELERATOR_ATTESTATION, + ) + + failed_result = run_llama.subprocess.CompletedProcess(["exporter"], 1, "", "query failed") + with mock.patch.object(run_llama.subprocess, "run", return_value=failed_result): + with self.assertRaisesRegex(preflight.PreflightError, "query failed"): + run_llama.query_accelerator_attestation(Path("/exporter"), "ROCm0") + + def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + xfs = root / "mnt" / "models" + btrfs = root / "home" + rotating = root / "rotating" + ram = root / "ram" + network = root / "network" + missing = root / "missing" + for directory in (xfs, btrfs, rotating, ram, network, missing): + directory.mkdir(parents=True) + (directory / "data").write_bytes(b"x") + + sys_root = root / "sys" + nvme1 = sys_root / "devices" / "pci" / "block" / "nvme1n1" + nvme0 = sys_root / "devices" / "pci" / "block" / "nvme0n1" + nvme0p3 = nvme0 / "nvme0n1p3" + sdb = sys_root / "devices" / "pci" / "block" / "sdb" + sdb1 = sdb / "sdb1" + for disk, rotational in ((nvme1, "0\n"), (nvme0, "0\n"), (sdb, "1\n")): + (disk / "queue").mkdir(parents=True) + (disk / "queue" / "rotational").write_text(rotational, encoding="ascii") + nvme0p3.mkdir() + sdb1.mkdir() + dev_block = sys_root / "dev" / "block" + dev_block.mkdir(parents=True) + (dev_block / "259:0").symlink_to(nvme1, target_is_directory=True) + (dev_block / "259:3").symlink_to(nvme0p3, target_is_directory=True) + (dev_block / "8:17").symlink_to(sdb1, target_is_directory=True) + class_block = sys_root / "class" / "block" + (class_block / "nvme0n1p3").mkdir(parents=True) + (class_block / "nvme0n1p3" / "dev").write_text("259:3\n", encoding="ascii") + + mountinfo = root / "mountinfo" + mountinfo.write_text( + f"1 0 259:0 / {xfs.resolve()} rw - xfs /dev/nvme1n1 rw\n" + f"2 0 0:35 /home {btrfs.resolve()} rw - btrfs /dev/nvme0n1p3[/home] rw\n" + f"3 0 8:17 / {rotating.resolve()} rw - ext4 /dev/sdb1 rw\n" + f"4 0 0:42 / {ram.resolve()} rw - tmpfs tmpfs rw\n" + f"5 0 0:43 / {network.resolve()} rw - nfs server:/share rw\n" + f"6 0 240:1 / {missing.resolve()} rw - ext4 /dev/missing rw\n", + encoding="ascii", + ) + + xfs_result = preflight.storage_attestation( + xfs / "data", + "xfs", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + self.assertEqual(xfs_result["filesystem_type"], "xfs") + self.assertEqual(xfs_result["nvme_device"], "nvme1n1") + + btrfs_result = preflight.storage_attestation( + btrfs / "new" / "trace", + "btrfs", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + self.assertEqual(btrfs_result["filesystem_type"], "btrfs") + self.assertEqual(btrfs_result["device_number"], "259:3") + self.assertEqual(btrfs_result["existing_path"], str(btrfs.resolve())) + self.assertEqual(btrfs_result["nvme_device"], "nvme0n1") + + for path, message in ( + (rotating / "data", "non-rotational"), + (ram / "data", "local block device"), + (network / "data", "local block device"), + (missing / "data", "cannot be resolved")): + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.storage_attestation( + path, + "invalid", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + + (xfs / "escape").symlink_to(ram, target_is_directory=True) + with self.assertRaisesRegex(preflight.PreflightError, "local block device"): + preflight.storage_attestation( + xfs / "escape" / "data", + "symlink escape", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + with self.assertRaisesRegex(preflight.PreflightError, "/mnt/bigspace"): + preflight.storage_attestation( + Path("/mnt/bigspace/model.gguf"), + "forbidden", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + + def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: + with mock.patch.object( + preflight, + "storage_attestation", + return_value=storage_record("/home/test")): + with mock.patch.dict(preflight.os.environ, {"HIP_LAUNCH_BLOCKING": "1"}, clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "TMPDIR is required"): + preflight.run_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + def test_watchdog_lease_rejects_arbitrary_heartbeat_process(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -916,6 +1122,20 @@ def test_rejects_unpinned_ds4_revision(self) -> None: with self.assertRaisesRegex(trace.TraceError, "ds4 revision"): trace.TraceBundle(root) + def test_rejects_unattested_accelerator_identity(self) -> None: + for key, value, message in ( + ("architecture", "gfx1100", "architecture mismatch"), + ("gfx_target_version", 110500, "gfx_target_version mismatch"), + ("pci_device_id", "ROCm0", "PCI identity")): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest() + trace_manifest["accelerator"][key] = value + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + def test_rejects_wrong_component_schema_and_same_bundle_compare(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index b54dea2d97ec..104ee2c4b854 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -12,3 +12,12 @@ target_compile_features(${PROMPT_TARGET} PRIVATE cxx_std_17) if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} ${PROMPT_TARGET} RUNTIME) endif() + +if(LLAMA_BUILD_TESTS) + set(HOST_TEST_TARGET test-deepseek41-trace-host) + add_executable(${HOST_TEST_TARGET} test-host-attestation.cpp) + target_link_libraries(${HOST_TEST_TARGET} PRIVATE ggml) + target_compile_features(${HOST_TEST_TARGET} PRIVATE cxx_std_17) + add_test(NAME ${HOST_TEST_TARGET} COMMAND $) + set_property(TEST ${HOST_TEST_TARGET} PROPERTY LABELS main) +endif() diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 2663e6b96164..8f6044bd9a7a 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -1,6 +1,6 @@ # DeepSeek V4.1 correctness traces -This directory defines the versioned cross-runtime trace format used by issue #48. It compares the unchanged published GGUF between llama.cpp and ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. +This directory defines version 2 of the cross-runtime trace format used by issue #48. It compares the unchanged published GGUF between llama.cpp and ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. Each trace is a directory: @@ -10,7 +10,7 @@ Each trace is a directory: - `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. - `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. -The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. +The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. Every bundle also carries an exact accelerator attestation. The selected backend device must map through its PCI identity and Linux KFD topology to `gfx_target_version=110501` (`gfx1151`); device labels or environment strings are not accepted as architecture evidence. Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, layer-0/1 raw attention-source rows as i32 `[128, token_count]`, compressed attention-source IDs as nonempty rank-2 i32 with width at most 512, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Raw attention rows use physical ring IDs `0..127`, visible current-ubatch IDs `128..128+token_index`, and unavailable sentinel `128+token_count`; layers 0 and 1 must be byte-identical for each execution step. Original expert IDs must be within `0..383`. @@ -59,7 +59,8 @@ cmake --build build-dsv41-trace-rocm --config Release -j "$(nproc)" --target \ test-deepseek41-engram \ test-deepseek41-expert \ test-deepseek41-memory \ - test-deepseek41-runtime + test-deepseek41-runtime \ + test-deepseek41-trace-host build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o MUL_MAT_ID build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o MUL_MAT @@ -68,7 +69,7 @@ build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o SET_ROWS build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o CPY ``` -Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly by default. +Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly, invokes the exact exporter for a pre-allocation device attestation, and rejects the run unless the backend PCI identity maps to exactly one KFD node reporting `gfx1151`. The native exporter repeats the query before model allocation and verifies that the loaded model still uses the same device. Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the complete matrix process group. The wrappers fail closed if this variable is absent or different, and every embedded memory, swap, and watchdog audit records it. Keep the same inherited value for ds4 and llama.cpp. @@ -80,7 +81,7 @@ The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd07479 The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. -Use one empty directory on verified non-rotational NVMe for every input and output. These metadata commands do not execute the model: +Use one empty directory on verified non-rotational NVMe for every input and output. The Python launchers and both native tools resolve symlinks and the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, and `/mnt/bigspace` fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory and has no `/tmp` fallback. These metadata commands do not execute the model: ```sh MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf @@ -110,13 +111,13 @@ The expected model digest is `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9 The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision and its build SHA-256 matches the executed file. -The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. +The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. The content-addressed memory audit binds the preflight accelerator and storage attestations; the manifest binds the independently repeated native accelerator attestation. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, and NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision or executable hash does not match that attestation. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision, executable hash, accelerator identity, or loaded model device does not match that attestation. `run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. -The external ds4 exporter is not present in the pinned `/home/papa/src/ds4-v41` checkout. It remains a blocker until a separately built executable is provided and attested. `run_ds4.py` currently has no approved exporter digest and fails closed before inference. After the exporter is implemented and reviewed, add its exact executable SHA-256 and pinned ds4 revision to `APPROVED_EXPORTERS` in `run_ds4.py`; a caller-provided digest alone is not sufficient oracle provenance. The exporter must accept the interface used by `run_ds4.py`: +The external ds4 exporter is not present in the pinned `/home/papa/src/ds4-v41` checkout. It remains a blocker until a separately built executable is provided and attested. `run_ds4.py` currently has no approved exporter digest and fails closed before inference. After the exporter is implemented and reviewed on an authorized oracle host, add its exact executable SHA-256 and pinned ds4 revision to `APPROVED_EXPORTERS` in `run_ds4.py`; a caller-provided digest alone is not sufficient oracle provenance. Its bundle must include the same KFD-derived `gfx1151` accelerator identity as the llama.cpp trace. The exporter must accept the interface used by `run_ds4.py`: The unpublished `ds4gguf` documentation revision `e13893ffcb33e90c8852929303e188102df7a8f5` is provenance only. It is not an executable dependency, exporter approval, or fixture source. Executable tests and fixtures stay in this `strix-llama.cpp` stack. diff --git a/tools/deepseek-v41-trace/host-attestation.h b/tools/deepseek-v41-trace/host-attestation.h new file mode 100644 index 000000000000..5476f3369549 --- /dev/null +++ b/tools/deepseek-v41-trace/host-attestation.h @@ -0,0 +1,327 @@ +#pragma once + +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dsv41 { + +namespace fs = std::filesystem; + +struct storage_attestation { + fs::path resolved_path; + fs::path existing_path; + std::string mount_point; + std::string filesystem_type; + std::string mount_source; + std::string device_number; + fs::path block_device_path; + std::string nvme_device; +}; + +struct accelerator_attestation { + std::string backend_device; + std::string backend_description; + std::string pci_device_id; + std::string kfd_node; + uint64_t gpu_id; + uint64_t gfx_target_version; + std::string architecture; +}; + +static inline std::string read_text(const fs::path & path) { + std::ifstream input(path); + if (!input) { + throw std::runtime_error("cannot read " + path.string()); + } + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +static inline std::string trim(std::string value) { + while (!value.empty() && std::isspace(static_cast(value.back()))) { + value.pop_back(); + } + size_t first = 0; + while (first < value.size() && std::isspace(static_cast(value[first]))) { + ++first; + } + return value.substr(first); +} + +static inline std::string decode_mount_field(const std::string & value) { + std::string result; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '\\' && i + 3 < value.size() && + value[i + 1] >= '0' && value[i + 1] <= '7' && + value[i + 2] >= '0' && value[i + 2] <= '7' && + value[i + 3] >= '0' && value[i + 3] <= '7') { + const int byte = (value[i + 1] - '0')*64 + (value[i + 2] - '0')*8 + value[i + 3] - '0'; + result.push_back(static_cast(byte)); + i += 3; + } else { + result.push_back(value[i]); + } + } + return result; +} + +static inline bool path_is_within(const fs::path & path, const fs::path & root) { + if (path == root) { + return true; + } + const fs::path relative = path.lexically_relative(root); + return !relative.empty() && *relative.begin() != ".."; +} + +static inline fs::path existing_ancestor(const fs::path & path) { + fs::path current = path; + while (!current.empty() && !fs::exists(current)) { + const fs::path parent = current.parent_path(); + if (parent == current) { + break; + } + current = parent; + } + if (current.empty() || !fs::exists(current)) { + throw std::runtime_error("path has no existing parent: " + path.string()); + } + return fs::canonical(current); +} + +static inline storage_attestation require_nvme_path( + const fs::path & path, + const char * label, + const fs::path & mountinfo_path = "/proc/self/mountinfo", + const fs::path & sys_dev_block_root = "/sys/dev/block", + const fs::path & sys_class_block_root = "/sys/class/block") { + const fs::path absolute = fs::absolute(path).lexically_normal(); + const fs::path resolved = fs::weakly_canonical(absolute); + const fs::path existing = existing_ancestor(resolved); + const fs::path forbidden = "/mnt/bigspace"; + if (path_is_within(resolved, forbidden)) { + throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); + } + + struct mount_record { + fs::path mount_point; + std::string filesystem_type; + std::string source; + std::string device_number; + }; + std::vector mounts; + std::istringstream mountinfo(read_text(mountinfo_path)); + std::string line; + while (std::getline(mountinfo, line)) { + std::istringstream fields_stream(line); + std::vector fields; + std::string field; + while (fields_stream >> field) { + fields.push_back(field); + } + const auto separator = std::find(fields.begin(), fields.end(), "-"); + if (fields.size() < 7 || separator == fields.end() || separator + 2 >= fields.end()) { + throw std::runtime_error("mountinfo contains an invalid record"); + } + mounts.push_back({ + decode_mount_field(fields[4]), + *(separator + 1), + decode_mount_field(*(separator + 2)), + fields[2], + }); + } + + const mount_record * selected = nullptr; + for (const mount_record & mount : mounts) { + if (path_is_within(existing, mount.mount_point) && + (selected == nullptr || mount.mount_point.string().size() > selected->mount_point.string().size())) { + selected = &mount; + } + } + if (selected == nullptr) { + throw std::runtime_error(std::string(label) + " mount cannot be resolved: " + resolved.string()); + } + + std::string device_number = selected->device_number; + if (device_number.substr(0, device_number.find(':')) == "0") { + const std::string source = selected->source.substr(0, selected->source.find('[')); + const fs::path source_path = source; + const fs::path source_dev = sys_class_block_root / source_path.filename() / "dev"; + if (source.rfind("/dev/", 0) != 0 || !fs::is_regular_file(source_dev)) { + throw std::runtime_error( + std::string(label) + " is not backed by a local block device: " + resolved.string()); + } + device_number = trim(read_text(source_dev)); + static const std::regex device_number_pattern("^[0-9]+:[0-9]+$"); + if (!std::regex_match(device_number, device_number_pattern)) { + throw std::runtime_error(std::string(label) + " backing device identity is invalid"); + } + } + + const fs::path device_link = sys_dev_block_root / device_number; + if (!fs::exists(device_link)) { + throw std::runtime_error( + std::string(label) + " is not backed by a resolvable block device: " + resolved.string()); + } + const fs::path block_device = fs::canonical(device_link); + fs::path rotational_path; + for (fs::path current = block_device; !current.empty(); current = current.parent_path()) { + const fs::path candidate = current / "queue" / "rotational"; + if (fs::is_regular_file(candidate)) { + rotational_path = candidate; + break; + } + if (current == current.parent_path()) { + break; + } + } + if (rotational_path.empty()) { + throw std::runtime_error( + std::string(label) + " block device rotational state cannot be resolved: " + block_device.string()); + } + if (trim(read_text(rotational_path)) != "0") { + throw std::runtime_error(std::string(label) + " must use non-rotational storage: " + resolved.string()); + } + + static const std::regex nvme_pattern("^nvme[0-9]+(c[0-9]+)?n[0-9]+$"); + std::string nvme_device; + for (const fs::path & component : block_device) { + const std::string name = component.string(); + if (std::regex_match(name, nvme_pattern)) { + nvme_device = name; + } + } + if (nvme_device.empty()) { + throw std::runtime_error( + std::string(label) + " must use an NVMe block device: " + block_device.string()); + } + + return { + resolved, + existing, + selected->mount_point.string(), + selected->filesystem_type, + selected->source, + device_number, + block_device, + nvme_device, + }; +} + +static inline std::map read_kfd_properties(const fs::path & path) { + std::map result; + std::istringstream input(read_text(path)); + std::string key; + uint64_t value = 0; + while (input >> key >> value) { + result[key] = value; + std::string rest; + std::getline(input, rest); + } + return result; +} + +static inline std::string gfx_architecture(uint64_t version) { + const uint64_t major = version / 10000; + const uint64_t minor = (version / 100) % 100; + const uint64_t stepping = version % 100; + if (major == 0 || minor > 9 || stepping > 9) { + throw std::runtime_error("KFD gfx_target_version is invalid"); + } + return "gfx" + std::to_string(major) + std::to_string(minor) + std::to_string(stepping); +} + +static inline accelerator_attestation require_gfx1151_identity( + const std::string & backend_device, + const std::string & backend_description, + const std::string & pci_device_id, + const fs::path & kfd_nodes_root) { + if (backend_device != "ROCm0") { + throw std::runtime_error("selected execution device must be ROCm0"); + } + if (backend_description.empty()) { + throw std::runtime_error("selected ROCm device description is missing"); + } + + static const std::regex pci_pattern("^([0-9a-f]{4}):([0-9a-f]{2}):([0-9a-f]{2})\\.([0-7])$"); + std::smatch match; + if (!std::regex_match(pci_device_id, match, pci_pattern)) { + throw std::runtime_error("selected ROCm device PCI identity is missing or invalid"); + } + const uint64_t domain = std::stoull(match[1].str(), nullptr, 16); + const uint64_t bus = std::stoull(match[2].str(), nullptr, 16); + const uint64_t slot = std::stoull(match[3].str(), nullptr, 16); + const uint64_t function = std::stoull(match[4].str(), nullptr, 16); + const uint64_t location_id = (bus << 8) | (slot << 3) | function; + + std::vector matches; + if (!fs::is_directory(kfd_nodes_root)) { + throw std::runtime_error("KFD topology is unavailable"); + } + for (const fs::directory_entry & entry : fs::directory_iterator(kfd_nodes_root)) { + if (!entry.is_directory()) { + continue; + } + const fs::path properties_path = entry.path() / "properties"; + const fs::path gpu_id_path = entry.path() / "gpu_id"; + if (!fs::is_regular_file(properties_path) || !fs::is_regular_file(gpu_id_path)) { + continue; + } + const std::map properties = read_kfd_properties(properties_path); + const auto domain_value = properties.find("domain"); + const auto location_value = properties.find("location_id"); + const auto gfx_value = properties.find("gfx_target_version"); + if (domain_value == properties.end() || location_value == properties.end() || + gfx_value == properties.end() || domain_value->second != domain || + location_value->second != location_id) { + continue; + } + const uint64_t gpu_id = std::stoull(trim(read_text(gpu_id_path))); + if (gpu_id == 0) { + throw std::runtime_error("selected KFD topology node has no GPU identity"); + } + matches.push_back({ + backend_device, + backend_description, + pci_device_id, + entry.path().filename().string(), + gpu_id, + gfx_value->second, + gfx_architecture(gfx_value->second), + }); + } + if (matches.size() != 1) { + throw std::runtime_error("selected ROCm device does not map to exactly one KFD topology node"); + } + if (matches[0].architecture != "gfx1151" || matches[0].gfx_target_version != 110501) { + throw std::runtime_error( + "selected ROCm device architecture must be gfx1151, found " + matches[0].architecture); + } + return matches[0]; +} + +static inline accelerator_attestation require_gfx1151_device( + ggml_backend_dev_t device, + const fs::path & kfd_nodes_root = "/sys/class/kfd/kfd/topology/nodes") { + if (device == nullptr) { + throw std::runtime_error("required ROCm device is unavailable"); + } + ggml_backend_dev_props props = {}; + ggml_backend_dev_get_props(device, &props); + return require_gfx1151_identity( + props.name == nullptr ? "" : props.name, + props.description == nullptr ? "" : props.description, + props.device_id == nullptr ? "" : props.device_id, + kfd_nodes_root); +} + +} diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 4923fd5f3c3b..32739dd72682 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -8,6 +8,7 @@ extern "C" { } #include "llama.h" #include "llama-ext.h" +#include "host-attestation.h" #include "trace-components.h" #include @@ -24,6 +25,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -42,7 +44,7 @@ extern "C" { namespace fs = std::filesystem; using json = nlohmann::ordered_json; -static constexpr int TRACE_VERSION = 1; +static constexpr int TRACE_VERSION = 2; #if defined(__linux__) static constexpr const char * WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f"; @@ -105,14 +107,6 @@ static std::vector read_file(const fs::path & path) { return result; } -static void require_nvme_path(const fs::path & path, const char * label) { - const fs::path absolute = fs::absolute(path).lexically_normal(); - const std::string value = absolute.string(); - if (value == "/mnt/bigspace" || value.rfind("/mnt/bigspace/", 0) == 0) { - throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); - } -} - static std::string required_environment(const char * name) { const char * value = std::getenv(name); if (value == nullptr || value[0] == '\0') { @@ -348,7 +342,7 @@ static void validate_watchdog(const json & data) { static json audit_reference(const char * environment_name, const char * expected_kind) { const fs::path path = required_environment(environment_name); - require_nvme_path(path, "audit"); + dsv41::require_nvme_path(path, "audit"); const std::vector bytes = read_file(path); json audit; try { @@ -382,6 +376,9 @@ static json audit_reference(const char * environment_name, const char * expected }; if (std::string(expected_kind) == "watchdog") { result["data"] = audit["data"]; + } else if (std::string(expected_kind) == "memory") { + result["accelerator"] = audit.value("accelerator", json::object()); + result["storage"] = audit.value("storage", json::object()); } return result; } @@ -623,9 +620,33 @@ static std::vector command_line(int argc, char ** argv) { return result; } +static json accelerator_json(const dsv41::accelerator_attestation & accelerator) { + return { + {"format", "dsv41-accelerator-attestation"}, + {"version", 1}, + {"backend_device", accelerator.backend_device}, + {"backend_description", accelerator.backend_description}, + {"pci_device_id", accelerator.pci_device_id}, + {"kfd_node", accelerator.kfd_node}, + {"gpu_id", accelerator.gpu_id}, + {"gfx_target_version", accelerator.gfx_target_version}, + {"architecture", accelerator.architecture}, + {"source", "linux-kfd-sysfs"}, + }; +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { + if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { + common_init(); + ggml_backend_load_all(); + const dsv41::accelerator_attestation accelerator = + dsv41::require_gfx1151_device(ggml_backend_dev_by_name(argv[2])); + std::cout << accelerator_json(accelerator).dump() << '\n'; + return 0; + } + const uint16_t endian = 1; if (*reinterpret_cast(&endian) != 1) { throw std::runtime_error("trace writer requires a little-endian host"); @@ -646,13 +667,24 @@ int main(int argc, char ** argv) { throw std::runtime_error("-n must request at least one deterministic decode step"); } - require_nvme_path(params.model.path, "model"); - require_nvme_path(params.prompt_file, "prompt"); - require_nvme_path(params.out_file, "trace output"); + const dsv41::storage_attestation model_storage = + dsv41::require_nvme_path(params.model.path, "model"); + const dsv41::storage_attestation prompt_storage = + dsv41::require_nvme_path(params.prompt_file, "prompt"); + const dsv41::storage_attestation output_storage = + dsv41::require_nvme_path(params.out_file, "trace output"); if (required_environment("HIP_LAUNCH_BLOCKING") != "1") { throw std::runtime_error("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs"); } + if (params.devices.size() != 1 || params.devices[0] == nullptr) { + throw std::runtime_error("trace tool requires exactly one selected execution device"); + } + const dsv41::accelerator_attestation configured_accelerator = + dsv41::require_gfx1151_device(params.devices[0]); const json memory_audit = audit_reference("DSV41_TRACE_MEMORY_AUDIT", "memory"); + if (memory_audit.value("accelerator", json::object()) != accelerator_json(configured_accelerator)) { + throw std::runtime_error("preflight accelerator audit does not match the selected execution device"); + } const json swap_audit = audit_reference("DSV41_TRACE_SWAP_AUDIT", "swap"); const json watchdog_audit = audit_reference("DSV41_TRACE_WATCHDOG_AUDIT", "watchdog"); @@ -662,9 +694,9 @@ int main(int argc, char ** argv) { throw std::runtime_error("parsed prompt differs from exact prompt file bytes"); } - const fs::path model_path = fs::absolute(params.model.path).lexically_normal(); - const fs::path prompt_path = fs::absolute(params.prompt_file).lexically_normal(); - const fs::path output_path = fs::absolute(params.out_file).lexically_normal(); + const fs::path model_path = model_storage.resolved_path; + const fs::path prompt_path = prompt_storage.resolved_path; + const fs::path output_path = output_storage.resolved_path; llama_backend_init(); llama_numa_init(params.numa); common_init_result_ptr init = common_init_from_params(params); @@ -683,6 +715,11 @@ int main(int argc, char ** argv) { if (model_devices != std::vector{"ROCm0"}) { throw std::runtime_error("trace tool requires the loaded model to use only ROCm0"); } + const dsv41::accelerator_attestation accelerator = + dsv41::require_gfx1151_device(llama_model_get_device(model, 0)); + if (accelerator_json(accelerator) != accelerator_json(configured_accelerator)) { + throw std::runtime_error("loaded model device differs from the pre-allocation accelerator attestation"); + } const llama_vocab * vocab = llama_model_get_vocab(model); const bool add_bos = llama_vocab_get_add_bos(vocab); const std::vector tokens = common_tokenize(ctx, params.prompt, add_bos, true); @@ -724,11 +761,14 @@ int main(int argc, char ** argv) { {"byte_count", prompt_bytes.size()}, {"sha256", sha256_data(prompt_bytes.data(), prompt_bytes.size())}, }}, + {"accelerator", accelerator_json(accelerator)}, {"config", { {"context", llama_n_ctx(ctx)}, {"batch", params.n_batch}, {"ubatch", params.n_ubatch}, {"device", model_devices[0]}, + {"device_architecture", accelerator.architecture}, + {"device_pci_id", accelerator.pci_device_id}, {"decode_steps", params.n_predict}, {"kv_type_k", ggml_type_name(params.cache_type_k)}, {"kv_type_v", ggml_type_name(params.cache_type_v)}, diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index ed232ca3e2d3..ad717bc20037 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -38,13 +38,137 @@ def resolved(path: Path) -> Path: return path.expanduser().resolve() -def require_nvme_path(path: Path, label: str) -> Path: +def _decode_mount_field(value: str) -> str: + return re.sub( + r"\\([0-7]{3})", + lambda match: chr(int(match.group(1), 8)), + value, + ) + + +def _existing_ancestor(path: Path) -> Path: + current = path + while not current.exists(): + if current == current.parent: + raise PreflightError(f"path has no existing parent: {path}") + current = current.parent + return current.resolve(strict=True) + + +def storage_attestation( + path: Path, + label: str, + *, + mountinfo_path: Path = Path("/proc/self/mountinfo"), + sys_dev_block_root: Path = Path("/sys/dev/block"), + sys_class_block_root: Path = Path("/sys/class/block")) -> dict[str, object]: path = resolved(path) try: path.relative_to(FORBIDDEN_ROOT) except ValueError: - return path - raise PreflightError(f"{label} must not use rotational storage: {path}") + pass + else: + raise PreflightError(f"{label} must not use /mnt/bigspace: {path}") + + existing = _existing_ancestor(path) + mounts: list[tuple[Path, str, str, str]] = [] + for line in read_proc_lines(mountinfo_path): + fields = line.split() + try: + separator = fields.index("-") + mount = ( + Path(_decode_mount_field(fields[4])), + fields[separator + 1], + _decode_mount_field(fields[separator + 2]), + fields[2], + ) + except (IndexError, ValueError) as error: + raise PreflightError(f"invalid mountinfo record: {line}") from error + try: + existing.relative_to(mount[0]) + except ValueError: + continue + mounts.append(mount) + if not mounts: + raise PreflightError(f"{label} mount cannot be resolved: {path}") + mount_point, filesystem_type, mount_source, device_number = max( + mounts, key=lambda item: len(item[0].parts)) + if device_number.split(":", 1)[0] == "0": + source_device = mount_source.split("[", 1)[0] + source_name = Path(source_device).name + source_dev_path = sys_class_block_root / source_name / "dev" + if not source_device.startswith("/dev/") or not source_dev_path.is_file(): + raise PreflightError(f"{label} is not backed by a local block device: {path}") + try: + device_number = source_dev_path.read_text(encoding="ascii").strip() + except OSError as error: + raise PreflightError(f"{label} backing device identity cannot be read: {error}") from error + if re.fullmatch(r"[0-9]+:[0-9]+", device_number) is None: + raise PreflightError(f"{label} backing device identity is invalid: {device_number}") + + device_link = sys_dev_block_root / device_number + if not device_link.is_symlink(): + raise PreflightError(f"{label} block device cannot be resolved: {path}") + try: + block_device = device_link.resolve(strict=True) + except OSError as error: + raise PreflightError(f"{label} block device cannot be resolved: {error}") from error + rotational_path = None + for candidate in (block_device, *block_device.parents): + path_candidate = candidate / "queue" / "rotational" + if path_candidate.is_file(): + rotational_path = path_candidate + break + if rotational_path is None: + raise PreflightError(f"{label} block device rotational state cannot be resolved: {block_device}") + try: + rotational = rotational_path.read_text(encoding="ascii").strip() + except OSError as error: + raise PreflightError(f"{label} block device rotational state cannot be read: {error}") from error + if rotational != "0": + raise PreflightError(f"{label} must use non-rotational storage: {path}") + + nvme_device = next( + (part for part in reversed(block_device.parts) if re.fullmatch(r"nvme[0-9]+(?:c[0-9]+)?n[0-9]+", part)), + None, + ) + if nvme_device is None: + raise PreflightError(f"{label} must use an NVMe block device: {block_device}") + return { + "resolved_path": str(path), + "existing_path": str(existing), + "mount_point": str(mount_point), + "filesystem_type": filesystem_type, + "mount_source": mount_source, + "device_number": device_number, + "block_device_path": str(block_device), + "nvme_device": nvme_device, + "rotational": False, + } + + +def require_nvme_path( + path: Path, + label: str, + *, + mountinfo_path: Path = Path("/proc/self/mountinfo"), + sys_dev_block_root: Path = Path("/sys/dev/block"), + sys_class_block_root: Path = Path("/sys/class/block")) -> Path: + attestation = storage_attestation( + path, + label, + mountinfo_path=mountinfo_path, + sys_dev_block_root=sys_dev_block_root, + sys_class_block_root=sys_class_block_root, + ) + return _attested_resolved_path(attestation, label) + + +def _attested_resolved_path(attestation: dict[str, object], label: str) -> Path: + resolved_path = attestation["resolved_path"] + if not isinstance(resolved_path, str): + raise PreflightError(f"{label} resolved path evidence is invalid") + return Path(resolved_path) def safe_trace_path(root: Path, relative: Path | str) -> Path: @@ -603,9 +727,17 @@ def run_preflight( repo: Path, busy_patterns: list[str], ) -> dict[str, object]: - model = require_nvme_path(model, "model") - prompt = require_nvme_path(prompt, "prompt") - output = require_nvme_path(output, "trace output") + model_storage = storage_attestation(model, "model") + prompt_storage = storage_attestation(prompt, "prompt") + output_storage = storage_attestation(output, "trace output") + repo_storage = storage_attestation(repo, "repository") + tmpdir_value = os.environ.get("TMPDIR") + if not tmpdir_value: + raise PreflightError("TMPDIR is required for NVMe-only correctness runs") + tmp_storage = storage_attestation(Path(tmpdir_value), "temporary directory") + model = _attested_resolved_path(model_storage, "model") + prompt = _attested_resolved_path(prompt_storage, "prompt") + output = _attested_resolved_path(output_storage, "trace output") if not model.is_file(): raise PreflightError(f"model is not a file: {model}") if not prompt.is_file(): @@ -629,6 +761,13 @@ def run_preflight( "watchdog": watchdog, "active_workloads": [], "environment": {"HIP_LAUNCH_BLOCKING": "1"}, + "storage": { + "model": model_storage, + "prompt": prompt_storage, + "output": output_storage, + "repository": repo_storage, + "temporary_directory": tmp_storage, + }, } @@ -674,6 +813,10 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: "data": data, "environment": audit["environment"], } + if key == "memory": + value["storage"] = audit["storage"] + if "accelerator" in audit: + value["accelerator"] = audit["accelerator"] path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") result[key] = str(path) summary = root / "preflight.json" diff --git a/tools/deepseek-v41-trace/prompt-builder.cpp b/tools/deepseek-v41-trace/prompt-builder.cpp index 82adbcd2d534..cfd66b98f0e4 100644 --- a/tools/deepseek-v41-trace/prompt-builder.cpp +++ b/tools/deepseek-v41-trace/prompt-builder.cpp @@ -1,4 +1,5 @@ #include "common.h" +#include "host-attestation.h" #include "llama.h" #include @@ -22,13 +23,6 @@ static std::string read_file(const fs::path & path) { return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); } -static void require_nvme_path(const fs::path & path, const char * label) { - const std::string value = fs::absolute(path).lexically_normal().string(); - if (value == "/mnt/bigspace" || value.rfind("/mnt/bigspace/", 0) == 0) { - throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); - } -} - static std::string argument(int argc, char ** argv, const std::string & name) { for (int index = 1; index + 1 < argc; ++index) { if (argv[index] == name) { @@ -48,16 +42,16 @@ static std::string model_architecture(const llama_model * model) { int main(int argc, char ** argv) { try { - const fs::path model_path = argument(argc, argv, "--model"); - const fs::path corpus_path = argument(argc, argv, "--corpus"); - const fs::path output_path = argument(argc, argv, "--output"); + fs::path model_path = argument(argc, argv, "--model"); + fs::path corpus_path = argument(argc, argv, "--corpus"); + fs::path output_path = argument(argc, argv, "--output"); const int64_t target_tokens = std::stoll(argument(argc, argv, "--tokens")); if (target_tokens < 2) { throw std::runtime_error("--tokens must be at least 2"); } - require_nvme_path(model_path, "model"); - require_nvme_path(corpus_path, "corpus"); - require_nvme_path(output_path, "prompt output"); + model_path = dsv41::require_nvme_path(model_path, "model").resolved_path; + corpus_path = dsv41::require_nvme_path(corpus_path, "corpus").resolved_path; + output_path = dsv41::require_nvme_path(output_path, "prompt output").resolved_path; if (fs::exists(output_path)) { throw std::runtime_error("prompt output already exists: " + output_path.string()); } diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index f313a4a58607..fd4aa5d2610d 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -4,6 +4,7 @@ import hashlib import json import os +import re import shlex import subprocess import sys @@ -93,18 +94,71 @@ def candidate_attestation(args: argparse.Namespace, exporter_sha256: str) -> dic def bind_candidate_attestation( output: Path, - attestation: dict[str, str]) -> None: + attestation: dict[str, str], + accelerator: dict[str, object]) -> None: manifest_path = safe_trace_path(output, "manifest.json") try: manifest = json.loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise PreflightError(f"cannot bind candidate attestation: {error}") from error + if manifest.get("accelerator") != accelerator: + raise PreflightError("llama trace accelerator attestation differs from the preflight query") manifest["candidate"] = attestation temp = manifest_path.with_suffix(".tmp") temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") os.replace(temp, manifest_path) +def validate_accelerator_attestation( + record: object, + *, + expected_device: str = "ROCm0") -> dict[str, object]: + if not isinstance(record, dict): + raise PreflightError("accelerator attestation is not an object") + expected = { + "format": "dsv41-accelerator-attestation", + "version": 1, + "backend_device": expected_device, + "architecture": "gfx1151", + "gfx_target_version": 110501, + "source": "linux-kfd-sysfs", + } + for key, value in expected.items(): + if record.get(key) != value: + raise PreflightError(f"accelerator attestation {key} mismatch") + if not isinstance(record.get("backend_description"), str) or not record["backend_description"]: + raise PreflightError("accelerator attestation backend description is missing") + pci_device_id = record.get("pci_device_id") + if not isinstance(pci_device_id, str) or re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", pci_device_id) is None: + raise PreflightError("accelerator attestation PCI identity is invalid") + if not isinstance(record.get("kfd_node"), str) or not record["kfd_node"].isdigit(): + raise PreflightError("accelerator attestation KFD node is invalid") + if not isinstance(record.get("gpu_id"), int) or record["gpu_id"] <= 0: + raise PreflightError("accelerator attestation GPU identity is invalid") + return dict(record) + + +def query_accelerator_attestation(exporter: Path, device: str) -> dict[str, object]: + try: + result = subprocess.run( + [str(exporter), "--dsv41-attest-device", device], + check=False, + capture_output=True, + text=True, + ) + except OSError as error: + raise PreflightError(f"cannot query selected accelerator: {error}") from error + if result.returncode != 0: + detail = result.stderr.strip() or f"exit {result.returncode}" + raise PreflightError(f"selected accelerator query failed: {detail}") + try: + record = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise PreflightError(f"selected accelerator query returned invalid JSON: {error}") from error + return validate_accelerator_attestation(record, expected_device=device) + + def build_command(args: argparse.Namespace, exporter: Path, output: Path) -> list[str]: return [ str(exporter), @@ -176,6 +230,10 @@ def main() -> int: validate_runtime_config(args) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") + exporter = resolved(args.exporter) + if not exporter.is_file() or not os.access(exporter, os.X_OK): + raise PreflightError(f"trace exporter is not executable: {exporter}") + accelerator = query_accelerator_attestation(exporter, args.device) if args.preflight_only: audit = run_preflight( model=args.model, @@ -184,12 +242,10 @@ def main() -> int: repo=args.repo, busy_patterns=args.busy_pattern, ) + audit["accelerator"] = accelerator print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) return 0 - exporter = resolved(args.exporter) - if not exporter.is_file() or not os.access(exporter, os.X_OK): - raise PreflightError(f"trace exporter is not executable: {exporter}") exporter_sha256 = sha256_file(exporter) model_sha256 = sha256_file(resolved(args.model)) if model_sha256 != MODEL_SHA256: @@ -214,12 +270,15 @@ def main() -> int: busy_patterns=args.busy_pattern, ) preflight_audit["runtime"] = "llama.cpp" + preflight_audit["accelerator"] = accelerator preflight_audit["config"] = { "context": args.context, "decode_steps": args.decode_steps, "batch": args.batch, "ubatch": args.ubatch, "device": args.device, + "device_architecture": accelerator["architecture"], + "device_pci_id": accelerator["pci_device_id"], "expert_cache_slots": args.expert_cache_slots, "expert_cache_mib": args.expert_cache_mib, "gpu_layers": args.gpu_layers, @@ -243,11 +302,15 @@ def main() -> int: repo=args.repo, busy_patterns=args.busy_pattern, ) + post_accelerator = query_accelerator_attestation(exporter, args.device) + if post_accelerator != accelerator: + raise PreflightError("selected accelerator identity changed during trace execution") postflight_audit["runtime"] = "llama.cpp" + postflight_audit["accelerator"] = post_accelerator post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) - bind_candidate_attestation(output, attestation) + bind_candidate_attestation(output, attestation, accelerator) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "llama.cpp": raise PreflightError("llama exporter wrote a non-llama.cpp trace") diff --git a/tools/deepseek-v41-trace/test-host-attestation.cpp b/tools/deepseek-v41-trace/test-host-attestation.cpp new file mode 100644 index 000000000000..dbe2ec349617 --- /dev/null +++ b/tools/deepseek-v41-trace/test-host-attestation.cpp @@ -0,0 +1,183 @@ +#include "host-attestation.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void write_file(const fs::path & path, const std::string & value) { + fs::create_directories(path.parent_path()); + std::ofstream output(path); + output << value; + if (!output) { + throw std::runtime_error("cannot write test fixture"); + } +} + +template +static void require_failure(Function function, const std::string & expected) { + try { + function(); + } catch (const std::runtime_error & error) { + if (std::string(error.what()).find(expected) == std::string::npos) { + throw; + } + return; + } + throw std::runtime_error("expected failure containing: " + expected); +} + +int main() { + const fs::path root = fs::temp_directory_path() / + ("dsv41-host-attestation-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + try { + const fs::path xfs = root / "mnt" / "models"; + const fs::path btrfs = root / "home"; + const fs::path rotating = root / "rotating"; + const fs::path ram = root / "ram"; + const fs::path network = root / "network"; + const fs::path missing = root / "missing"; + fs::create_directories(xfs); + fs::create_directories(btrfs); + fs::create_directories(rotating); + fs::create_directories(ram); + fs::create_directories(network); + fs::create_directories(missing); + write_file(xfs / "model.gguf", "model"); + write_file(btrfs / "corpus.txt", "corpus"); + write_file(rotating / "data", "data"); + write_file(ram / "data", "data"); + write_file(network / "data", "data"); + write_file(missing / "data", "data"); + fs::create_directory_symlink(ram, xfs / "escape"); + + const fs::path sys = root / "sys"; + const fs::path nvme1 = sys / "devices" / "pci" / "block" / "nvme1n1"; + const fs::path nvme0 = sys / "devices" / "pci" / "block" / "nvme0n1"; + const fs::path nvme0p3 = nvme0 / "nvme0n1p3"; + const fs::path sdb = sys / "devices" / "pci" / "block" / "sdb"; + const fs::path sdb1 = sdb / "sdb1"; + write_file(nvme1 / "queue" / "rotational", "0\n"); + write_file(nvme0 / "queue" / "rotational", "0\n"); + write_file(sdb / "queue" / "rotational", "1\n"); + fs::create_directories(nvme0p3); + fs::create_directories(sdb1); + fs::create_directories(sys / "dev" / "block"); + fs::create_directory_symlink(nvme1, sys / "dev" / "block" / "259:0"); + fs::create_directory_symlink(nvme0p3, sys / "dev" / "block" / "259:3"); + fs::create_directory_symlink(sdb1, sys / "dev" / "block" / "8:17"); + write_file(sys / "class" / "block" / "nvme0n1p3" / "dev", "259:3\n"); + + const fs::path mountinfo = root / "mountinfo"; + write_file( + mountinfo, + "1 0 259:0 / " + fs::canonical(xfs).string() + " rw - xfs /dev/nvme1n1 rw\n" + + "2 0 0:35 /home " + fs::canonical(btrfs).string() + " rw - btrfs /dev/nvme0n1p3[/home] rw\n" + + "3 0 8:17 / " + fs::canonical(rotating).string() + " rw - ext4 /dev/sdb1 rw\n" + + "4 0 0:42 / " + fs::canonical(ram).string() + " rw - tmpfs tmpfs rw\n" + + "5 0 0:43 / " + fs::canonical(network).string() + " rw - nfs server:/share rw\n" + + "6 0 240:1 / " + fs::canonical(missing).string() + " rw - ext4 /dev/missing rw\n"); + + const dsv41::storage_attestation xfs_attestation = + dsv41::require_nvme_path(xfs / "model.gguf", "model", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + if (xfs_attestation.filesystem_type != "xfs" || xfs_attestation.nvme_device != "nvme1n1") { + throw std::runtime_error("xfs NVMe attestation mismatch"); + } + const dsv41::storage_attestation btrfs_attestation = + dsv41::require_nvme_path(btrfs / "new" / "trace", "trace", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + if (btrfs_attestation.filesystem_type != "btrfs" || + btrfs_attestation.device_number != "259:3" || + btrfs_attestation.nvme_device != "nvme0n1") { + throw std::runtime_error("btrfs NVMe partition attestation mismatch"); + } + require_failure( + [&]() { + dsv41::require_nvme_path( + rotating / "data", "rotating", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "non-rotational"); + require_failure( + [&]() { + dsv41::require_nvme_path( + ram / "data", "tmpfs", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "local block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + network / "data", "network", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "local block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + missing / "data", "missing", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "resolvable block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + xfs / "escape" / "data", "symlink escape", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + }, + "local block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + "/mnt/bigspace/model.gguf", "forbidden", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + }, + "/mnt/bigspace"); + + const fs::path kfd = root / "kfd"; + write_file( + kfd / "1" / "properties", + "domain 0\nlocation_id 50688\ngfx_target_version 110501\n"); + write_file(kfd / "1" / "gpu_id", "1234\n"); + const dsv41::accelerator_attestation accelerator = + dsv41::require_gfx1151_identity( + "ROCm0", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + if (accelerator.architecture != "gfx1151" || accelerator.gfx_target_version != 110501) { + throw std::runtime_error("gfx1151 attestation mismatch"); + } + require_failure( + [&]() { + dsv41::require_gfx1151_identity( + "ROCm1", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + }, + "ROCm0"); + write_file( + kfd / "1" / "properties", + "domain 0\nlocation_id 50688\ngfx_target_version 110500\n"); + require_failure( + [&]() { + dsv41::require_gfx1151_identity( + "ROCm0", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + }, + "gfx1151"); + write_file( + kfd / "1" / "properties", + "domain 0\nlocation_id 50688\n"); + require_failure( + [&]() { + dsv41::require_gfx1151_identity( + "ROCm0", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + }, + "exactly one KFD"); + + fs::remove_all(root); + return 0; + } catch (const std::exception & error) { + std::cerr << "test-host-attestation: " << error.what() << '\n'; + std::error_code ec; + fs::remove_all(root, ec); + return 1; + } +} diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index b8fe07ac4c63..a3eb67307cd6 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -12,7 +12,7 @@ from typing import Any, BinaryIO, Iterable TRACE_FORMAT = "dsv41-trace" -TRACE_VERSION = 1 +TRACE_VERSION = 2 DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" MODEL_SHA256 = "1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42" REPOSITORY = "halo-box/strix-llama.cpp" @@ -388,7 +388,9 @@ def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: return result def _validate_manifest(self) -> None: - for key in ("runtime", "revision", "build", "model", "prompt", "config", "comparison", "environment", "audits"): + for key in ( + "runtime", "revision", "build", "model", "prompt", "accelerator", + "config", "comparison", "environment", "audits"): if key not in self.manifest: raise TraceError(f"manifest is missing {key}") if not isinstance(self.manifest["runtime"], str) or not self.manifest["runtime"]: @@ -424,6 +426,31 @@ def _validate_manifest(self) -> None: raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") if self.manifest["model"].get("architecture") != "deepseek41": raise TraceError("model architecture must be deepseek41") + accelerator = self.manifest["accelerator"] + if not isinstance(accelerator, dict): + raise TraceError("manifest accelerator attestation is invalid") + accelerator_expected = { + "format": "dsv41-accelerator-attestation", + "version": 1, + "architecture": "gfx1151", + "gfx_target_version": 110501, + "source": "linux-kfd-sysfs", + } + for key, value in accelerator_expected.items(): + if accelerator.get(key) != value: + raise TraceError(f"manifest accelerator {key} mismatch") + if not isinstance(accelerator.get("backend_description"), str) or not accelerator["backend_description"]: + raise TraceError("manifest accelerator backend description is invalid") + if not isinstance(accelerator.get("backend_device"), str) or not accelerator["backend_device"]: + raise TraceError("manifest accelerator backend device is invalid") + if re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", + accelerator.get("pci_device_id", "")) is None: + raise TraceError("manifest accelerator PCI identity is invalid") + if not isinstance(accelerator.get("kfd_node"), str) or not accelerator["kfd_node"].isdigit(): + raise TraceError("manifest accelerator KFD node is invalid") + if not isinstance(accelerator.get("gpu_id"), int) or accelerator["gpu_id"] <= 0: + raise TraceError("manifest accelerator GPU identity is invalid") corpus_name = self.manifest["prompt"].get("corpus_name") if corpus_name not in CORPUS_SHA256: raise TraceError("prompt corpus is not in the fixed correctness corpus set") @@ -500,6 +527,8 @@ def _validate_manifest(self) -> None: raise TraceError("logit comparison policy must be byte-identical-f32") config = self.manifest["config"] if self.manifest["runtime"] == "llama.cpp": + if accelerator["backend_device"] != "ROCm0": + raise TraceError("llama.cpp accelerator backend device must be ROCm0") if config.get("batch") != ADMITTED_BATCH or config.get("ubatch") != ADMITTED_UBATCH: raise TraceError("llama.cpp trace does not use the admitted batch and ubatch") if config.get("expert_cache_slots") != REQUIRED_EXPERT_SLOTS or ( @@ -507,6 +536,9 @@ def _validate_manifest(self) -> None: raise TraceError("llama.cpp trace does not use the admitted expert cache") if config.get("device") != "ROCm0" or config.get("gpu_layers") != 99: raise TraceError("llama.cpp trace does not use the required ROCm0 offload") + if config.get("device_architecture") != accelerator["architecture"] or ( + config.get("device_pci_id") != accelerator["pci_device_id"]): + raise TraceError("llama.cpp trace device identity is not bound to the accelerator attestation") if config.get("kv_type_k") != "f16" or config.get("kv_type_v") != "f16" or ( config.get("flash_attention") not in (True, 1)) or config.get("load_mode") != 0: raise TraceError("llama.cpp trace inference configuration is invalid") @@ -554,6 +586,40 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: used = record["data"].get("mem_used_bytes") if not isinstance(used, int) or used < 0 or used >= SOFT_MEMORY_LIMIT: raise TraceError(f"{phase} memory audit evidence is invalid") + storage = record.get("storage") + required_storage = { + "model", "prompt", "output", "repository", "temporary_directory", + } + if not isinstance(storage, dict) or set(storage) != required_storage: + raise TraceError(f"{phase} memory audit storage evidence is missing") + for label, item in storage.items(): + if not isinstance(label, str) or not isinstance(item, dict): + raise TraceError(f"{phase} memory audit storage evidence is invalid") + if item.get("rotational") is not False or not isinstance(item.get("nvme_device"), str): + raise TraceError(f"{phase} memory audit storage is not non-rotational NVMe") + if re.fullmatch(r"nvme[0-9]+(?:c[0-9]+)?n[0-9]+", item["nvme_device"]) is None: + raise TraceError(f"{phase} memory audit NVMe device identity is invalid") + for path_key in ("resolved_path", "existing_path", "mount_point", "block_device_path"): + value = item.get(path_key) + if not isinstance(value, str) or not value.startswith("/"): + raise TraceError(f"{phase} memory audit {path_key} is invalid") + if not isinstance(item.get("filesystem_type"), str) or not item["filesystem_type"]: + raise TraceError(f"{phase} memory audit filesystem type is invalid") + if not isinstance(item.get("mount_source"), str) or not item["mount_source"]: + raise TraceError(f"{phase} memory audit mount source is invalid") + if not item["mount_source"].startswith("/dev/"): + raise TraceError(f"{phase} memory audit mount source is not a local block device") + if re.fullmatch(r"[0-9]+:[0-9]+", item.get("device_number", "")) is None: + raise TraceError(f"{phase} memory audit device number is invalid") + if item["nvme_device"] not in Path(item["block_device_path"]).parts: + raise TraceError(f"{phase} memory audit block device ancestry is invalid") + try: + Path(item["resolved_path"]).relative_to(Path(item["mount_point"])) + Path(item["existing_path"]).relative_to(Path(item["mount_point"])) + except ValueError as error: + raise TraceError(f"{phase} memory audit mount ancestry is invalid") from error + if self.manifest["runtime"] == "llama.cpp" and record.get("accelerator") != self.manifest["accelerator"]: + raise TraceError(f"{phase} memory audit accelerator evidence mismatch") if kind == "swap": if record["data"].get("enabled") is not False or record["data"].get("entries") != []: raise TraceError(f"{phase} swap audit evidence does not report zero configured swap") @@ -886,6 +952,8 @@ def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: checks = ( ("model.sha256", "model_identity"), ("model.architecture", "model_identity"), + ("accelerator.architecture", "accelerator_identity"), + ("accelerator.pci_device_id", "accelerator_identity"), ("prompt.sha256", "prompt_identity"), ("prompt.byte_count", "prompt_identity"), ("expected.prompt_tokens", "tokenizer"), From 11b18e371a5708c7f80a7c37001352e36253527d Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 02:23:30 -0700 Subject: [PATCH 21/56] trace : split runtime host attestations Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 713 +++++++++++++++- tools/deepseek-v41-trace/README.md | 65 +- tools/deepseek-v41-trace/host-attestation.h | 30 +- tools/deepseek-v41-trace/llama-trace.cpp | 43 +- tools/deepseek-v41-trace/preflight.py | 420 +++++++++- tools/deepseek-v41-trace/prompt-builder.cpp | 9 + tools/deepseek-v41-trace/run_ds4.py | 231 +++++- tools/deepseek-v41-trace/run_llama.py | 41 +- tools/deepseek-v41-trace/run_matrix.py | 78 +- .../test-host-attestation.cpp | 17 + tools/deepseek-v41-trace/trace_format.py | 764 +++++++++++++++--- 11 files changed, 2130 insertions(+), 281 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 52bd2072706b..0bd366dbcd14 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import importlib.util +import io import json import struct import sys @@ -28,16 +29,35 @@ { "timestamp": "1970-01-01T00:00:01.000Z", "event": "preflight", + "total_bytes": 128 * 1024 * 1024 * 1024, + "available_bytes": 64 * 1024 * 1024 * 1024, + "used_bytes": 64 * 1024 * 1024 * 1024, + "swap_entries": 0, + "peak_used_bytes": 64 * 1024 * 1024 * 1024, + "child_pid": None, + "child_status": "not_started", + "child_returncode": None, + "process_group_id": None, + "process_group_status": "not_created", + "threshold_reason": "none", "soft_bytes": trace.SOFT_MEMORY_LIMIT, "emergency_bytes": trace.WATCHDOG_EMERGENCY_LIMIT, "strict_ceiling_bytes": trace.STRICT_MEMORY_LIMIT, - "swap_entries": 0, }, { "timestamp": "1970-01-01T00:00:01.000Z", "event": "child_started", + "total_bytes": 128 * 1024 * 1024 * 1024, + "available_bytes": 64 * 1024 * 1024 * 1024, + "used_bytes": 64 * 1024 * 1024 * 1024, + "swap_entries": 0, + "peak_used_bytes": 64 * 1024 * 1024 * 1024, "child_pid": 456, + "child_status": "running", + "child_returncode": None, "process_group_id": 455, + "process_group_status": "active", + "threshold_reason": "none", "command": ["python3", "run_matrix.py"], }, ] @@ -49,7 +69,10 @@ ACCELERATOR_ATTESTATION = { "format": "dsv41-accelerator-attestation", - "version": 1, + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "backend": "ROCm", "backend_device": "ROCm0", "backend_description": "AMD Radeon Graphics", "pci_device_id": "0000:c1:00.0", @@ -60,6 +83,22 @@ "source": "linux-kfd-sysfs", } +METAL_ACCELERATOR_ATTESTATION = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "backend": "Metal", + "backend_device": "Metal0", + "backend_description": "Apple M3 Ultra", + "architecture": "Apple M3 Ultra", + "metal_registry_id": 0x12345678, + "recommended_max_working_set_bytes": 256 * 1024 * 1024 * 1024, + "unified_memory": True, + "source": "metal-device-query", +} + + def storage_record(path: str) -> dict[str, object]: model_storage = path.startswith("/mnt/models") mount_point = "/mnt/models" if model_storage else "/home" @@ -67,6 +106,11 @@ def storage_record(path: str) -> dict[str, object]: device_number = "259:0" if model_storage else "259:3" nvme_device = "nvme1n1" if model_storage else "nvme0n1" return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "storage_kind": "linux-nvme", "resolved_path": path, "existing_path": path, "mount_point": mount_point, @@ -76,6 +120,7 @@ def storage_record(path: str) -> dict[str, object]: "block_device_path": f"/sys/devices/pci/block/{nvme_device}", "nvme_device": nvme_device, "rotational": False, + "source": "linux-mountinfo-sysfs", } @@ -87,6 +132,70 @@ def storage_record(path: str) -> dict[str, object]: "temporary_directory": storage_record("/home/tmp"), } +def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, object]: + return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "storage_kind": "darwin-local-solid-state", + "resolved_path": path, + "existing_path": path, + "mount_point": mount_point, + "filesystem_type": "apfs", + "device_identifier": "disk3s1", + "parent_whole_disk": "disk3", + "bus_protocol": "Apple Fabric", + "filesystem_device": 1, + "internal": True, + "solid_state": True, + "source": "diskutil-info-plist", + } + + +DS4_STORAGE_ATTESTATION = { + "model": metal_storage_record("/Users/oracle/model.gguf"), + "prompt": metal_storage_record("/Users/oracle/prompt.txt"), + "output": metal_storage_record("/Users/oracle/output"), + "repository": metal_storage_record("/Users/oracle/repo"), + "runtime_checkout": metal_storage_record("/Users/oracle/ds4"), + "temporary_directory": metal_storage_record("/Users/oracle/tmp"), + "runner_executable": metal_storage_record("/usr/bin/python3", "/"), + "runner_script": metal_storage_record("/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py"), + "exporter": metal_storage_record("/Users/oracle/bin/ds4-trace"), +} + +DS4_HOST_ATTESTATION = { + "format": "dsv41-host-attestation", + "version": 1, + "runtime_kind": "apple-metal", + "platform": "macos", + "machine": "arm64", + "hardware_model": "Mac14,8", + "os_version": "15.6", + "memory_bytes": 256 * 1024 * 1024 * 1024, + "source": "darwin-sysctl", +} + +DS4_RUNNER_ATTESTATION = { + "format": "dsv41-runner-ownership", + "version": 1, + "runtime_kind": "apple-metal", + "source": "python-subprocess", + "runner_pid": 100, + "runner_parent_pid": 99, + "runner_uid": 501, + "runner_executable": "/usr/bin/python3", + "runner_executable_sha256": "1" * 64, + "runner_script": "/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py", + "runner_script_sha256": "2" * 64, + "exporter_path": "/Users/oracle/bin/ds4-trace", + "exporter_sha256": "3" * 64, + "checkout_path": "/Users/oracle/ds4", + "checkout_revision": trace.DS4_REVISION, + "command_sha256": "4" * 64, +} + AUDIT_RECORDS = { "memory": { "created_unix": 1, @@ -148,14 +257,66 @@ def storage_record(path: str) -> dict[str, object]: }, } +DS4_AUDIT_RECORDS = { + "memory": { + "created_unix": 1, + "kind": "memory", + "environment": {}, + "data": { + "mem_total_bytes": DS4_HOST_ATTESTATION["memory_bytes"], + "mem_available_bytes": 128 * 1024 * 1024 * 1024, + "mem_used_bytes": 128 * 1024 * 1024 * 1024, + }, + "storage": DS4_STORAGE_ATTESTATION, + "accelerator": dict(METAL_ACCELERATOR_ATTESTATION), + "host": DS4_HOST_ATTESTATION, + }, + "swap": { + "created_unix": 1, + "kind": "swap", + "environment": {}, + "data": { + "source": "darwin-sysctl-vm.swapusage", + "total_bytes": 0, + "used_bytes": 0, + "free_bytes": 0, + }, + }, + "runner": { + "created_unix": 1, + "kind": "runner", + "environment": {}, + "data": DS4_RUNNER_ATTESTATION, + }, +} + -def audit_bytes(kind: str, phase: str) -> bytes: - record = json.loads(json.dumps(AUDIT_RECORDS[kind])) +def audit_bytes(kind: str, phase: str, runtime: str = "llama.cpp") -> bytes: + records = DS4_AUDIT_RECORDS if runtime == "ds4" else AUDIT_RECORDS + record = json.loads(json.dumps(records[kind])) if kind == "watchdog": record["data"]["audit"]["path"] = f"audits/{phase}/{WATCHDOG_JSONL_SHA256}.jsonl" return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") +def replace_audit_record(root: Path, phase: str, kind: str, record: dict[str, object]) -> None: + data = (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + digest = trace.sha256_bytes(data) + path = root / "audits" / phase / f"{digest}.json" + path.write_bytes(data) + manifest_path = root / trace.MANIFEST_NAME + manifest_record = json.loads(manifest_path.read_text(encoding="ascii")) + manifest_record["audits"][phase][kind] = { + "path": f"audits/{phase}/{digest}.json", + "sha256": digest, + "created_unix": record["created_unix"], + } + manifest_path.write_text( + json.dumps(manifest_record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + + def provenance_bytes(prompt: bytes = b"abc") -> bytes: record = { "format": "dsv41-prompt-provenance", @@ -174,13 +335,38 @@ def provenance_bytes(prompt: bytes = b"abc") -> bytes: def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: provenance_sha256 = trace.sha256_bytes(provenance_bytes(prompt)) + is_ds4 = runtime == "ds4" + storage = DS4_STORAGE_ATTESTATION if is_ds4 else STORAGE_ATTESTATION + audit_kinds = ("memory", "swap", "runner") if is_ds4 else ("memory", "swap", "watchdog") result = { "runtime": runtime, - "revision": trace.DS4_REVISION if runtime == "ds4" else "a" * 40, - "build": {"sha256": "3" * 64}, - "model": {"sha256": trace.MODEL_SHA256, "byte_count": 123, "architecture": "deepseek41"}, - "accelerator": dict(ACCELERATOR_ATTESTATION), + "revision": trace.DS4_REVISION if is_ds4 else "a" * 40, + "build": ( + { + "compiler": "clang", + "target": "arm64-apple-darwin", + "path": "/Users/oracle/bin/ds4-trace", + "sha256": "3" * 64, + } + if is_ds4 + else { + "number": 1, + "info": "test", + "compiler": "clang", + "target": "arm64-apple-darwin", + "path": "/home/repo/build/bin/llama-deepseek-v41-trace", + "sha256": "3" * 64, + } + ), + "model": { + "path": storage["model"]["resolved_path"], + "sha256": trace.MODEL_SHA256, + "byte_count": 123, + "architecture": "deepseek41", + }, + "accelerator": dict(METAL_ACCELERATOR_ATTESTATION if is_ds4 else ACCELERATOR_ATTESTATION), "prompt": { + "path": storage["prompt"]["resolved_path"], "sha256": trace.sha256_bytes(prompt), "byte_count": len(prompt), "corpus_name": "correctness-prose.txt", @@ -194,18 +380,6 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "config": { "context": 3, "decode_steps": 1, - "batch": trace.ADMITTED_BATCH, - "ubatch": trace.ADMITTED_UBATCH, - "kv_type_k": "f16", - "kv_type_v": "f16", - "flash_attention": True, - "expert_cache_slots": trace.REQUIRED_EXPERT_SLOTS, - "expert_cache_bytes": trace.REQUIRED_EXPERT_CACHE_BYTES, - "device": "ROCm0", - "device_architecture": "gfx1151", - "device_pci_id": "0000:c1:00.0", - "gpu_layers": 99, - "load_mode": 0, "deepseek41": { "layer_count": 40, "vocab_size": 129280, @@ -222,7 +396,18 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "candidate_propagation_layers": [24, 28, 32, 36], }, }, - "comparison": {"logits": "byte-identical-f32"}, + "paths": { + label: record["resolved_path"] + for label, record in storage.items() + }, + "comparison": { + "tokens": "exact", + "engram_rows": "exact", + "expert_ids": "exact-original-id-space", + "expert_weights": "byte-identical-f32", + "attention_candidates": "exact", + "logits": "byte-identical-f32", + }, "expected": { "prompt_tokens": 2, "decode_steps": 1, @@ -240,20 +425,39 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "decode.greedy_token": {"layers": None, "decode": "steps"}, }, }, - "environment": {}, + "environment": { + "system_info": "Linux test system" if runtime == "llama.cpp" else "macOS test system", + "command": "test command", + }, "audits": { phase: { kind: { - "path": f"audits/{phase}/{trace.sha256_bytes(audit_bytes(kind, phase))}.json", - "sha256": trace.sha256_bytes(audit_bytes(kind, phase)), + "path": f"audits/{phase}/{trace.sha256_bytes(audit_bytes(kind, phase, runtime))}.json", + "sha256": trace.sha256_bytes(audit_bytes(kind, phase, runtime)), "created_unix": 1, } - for kind in ("memory", "swap", "watchdog") + for kind in audit_kinds } for phase in ("pre", "post") }, } - if runtime == "llama.cpp": + if not is_ds4: + result["config"].update({ + "batch": trace.ADMITTED_BATCH, + "ubatch": trace.ADMITTED_UBATCH, + "kv_type_k": "f16", + "kv_type_v": "f16", + "flash_attention": True, + "expert_cache_slots": trace.REQUIRED_EXPERT_SLOTS, + "expert_cache_bytes": trace.REQUIRED_EXPERT_CACHE_BYTES, + "device": "ROCm0", + "device_architecture": "gfx1151", + "device_pci_id": "0000:c1:00.0", + "gpu_layers": 99, + "load_mode": 0, + "tokenizer_add_bos": True, + "tokenizer_parse_special": True, + }) result["candidate"] = { "repository": trace.REPOSITORY, "revision": "a" * 40, @@ -262,18 +466,24 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "executable_sha256": "3" * 64, } else: + result["host"] = dict(DS4_HOST_ATTESTATION) result["config"]["prefill_chunk"] = trace.ADMITTED_UBATCH + result["config"]["device_backend"] = "Metal" + result["config"]["device_registry_id"] = METAL_ACCELERATOR_ATTESTATION["metal_registry_id"] return result def add_required_events(writer: object, logits: bytes | None = None, prompt: bytes = b"abc") -> None: + runtime = writer.manifest["runtime"] + audit_kinds = ("memory", "swap", "runner") if runtime == "ds4" else ("memory", "swap", "watchdog") for phase in ("pre", "post"): audit_root = writer.root / "audits" / phase audit_root.mkdir(parents=True, exist_ok=True) - for kind in ("memory", "swap", "watchdog"): - data = audit_bytes(kind, phase) + for kind in audit_kinds: + data = audit_bytes(kind, phase, runtime) (audit_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) - (audit_root / f"{WATCHDOG_JSONL_SHA256}.jsonl").write_bytes(WATCHDOG_JSONL) + if runtime == "llama.cpp": + (audit_root / f"{WATCHDOG_JSONL_SHA256}.jsonl").write_bytes(WATCHDOG_JSONL) provenance_root = writer.root / "provenance" provenance_root.mkdir(exist_ok=True) data = provenance_bytes(prompt) @@ -671,7 +881,7 @@ def test_accelerator_attestation_rejects_wrong_missing_and_spoofed_architecture( ) for key, value, message in ( ("architecture", "gfx1100", "architecture mismatch"), - ("architecture", None, "architecture mismatch"), + ("architecture", None, "fields are invalid"), ("backend_device", "ROCm1", "backend_device mismatch"), ("gfx_target_version", 110500, "gfx_target_version mismatch"), ("source", "environment", "source mismatch")): @@ -703,6 +913,30 @@ def test_accelerator_query_fails_closed(self) -> None: with self.assertRaisesRegex(preflight.PreflightError, "query failed"): run_llama.query_accelerator_attestation(Path("/exporter"), "ROCm0") + def test_metal_accelerator_attestation_is_runtime_specific(self) -> None: + valid = dict(METAL_ACCELERATOR_ATTESTATION) + self.assertEqual(run_ds4.validate_accelerator_attestation(valid), valid) + for mutation, message in ( + ({"runtime_kind": "strix-rocm"}, "runtime_kind mismatch"), + ({"platform": "linux"}, "platform mismatch"), + ({"backend": "ROCm"}, "backend mismatch"), + ({"source": "environment"}, "source mismatch"), + ({"pci_device_id": "0000:c1:00.0"}, "fields are invalid")): + invalid = dict(valid) + invalid.update(mutation) + with self.assertRaisesRegex(preflight.PreflightError, message): + run_ds4.validate_accelerator_attestation(invalid) + + def test_metal_accelerator_query_rejects_duplicate_keys(self) -> None: + duplicate = json.dumps(METAL_ACCELERATOR_ATTESTATION).replace( + '"backend": "Metal"', + '"backend": "Metal", "backend": "Metal"', + ) + result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, duplicate, "") + with mock.patch.object(run_ds4.subprocess, "run", return_value=result): + with self.assertRaisesRegex(preflight.PreflightError, "duplicate JSON key"): + run_ds4.query_accelerator_attestation(Path("/exporter"), "Metal0") + def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -800,6 +1034,140 @@ def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: sys_dev_block_root=dev_block, sys_class_block_root=class_block, ) + forbidden = root / "forbidden" + forbidden.mkdir() + (forbidden / "escape").symlink_to(xfs, target_is_directory=True) + with self.assertRaisesRegex(preflight.PreflightError, "must not use"): + preflight.storage_attestation( + forbidden / "escape" / "model.gguf", + "forbidden symlink", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + forbidden_root=forbidden, + ) + + def test_darwin_storage_and_host_preflight_are_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for name in ("repo", "ds4", "tmp", "output"): + (root / name).mkdir() + model = root / "model.gguf" + prompt = root / "prompt.txt" + model.write_bytes(b"model") + prompt.write_bytes(b"prompt") + runner_executable = root / "python3" + runner_script = root / "repo" / "tools" / "deepseek-v41-trace" / "run_ds4.py" + exporter = root / "ds4-trace" + runner_script.parent.mkdir(parents=True) + runner_executable.write_bytes(b"python") + runner_script.write_bytes(b"runner") + exporter.write_bytes(b"exporter") + + def disk_info(path: Path) -> dict[str, object]: + return { + "MountPoint": str(root.resolve()), + "FilesystemType": "apfs", + "DeviceIdentifier": "disk3s1", + "ParentWholeDisk": "disk3", + "BusProtocol": "Apple Fabric", + "Internal": True, + "SolidState": True, + "VolumeNetwork": False, + "DiskImage": False, + } + + def command_text(*args: str) -> str: + commands = { + ("sysctl", "-n", "hw.memsize"): str(256 * 1024 * 1024 * 1024), + ("sysctl", "-n", "hw.model"): "Mac14,8", + ("sysctl", "-n", "kern.osproductversion"): "15.6", + ("sysctl", "-n", "vm.swapusage"): "total = 0.00M used = 0.00M free = 0.00M (encrypted)", + ("vm_stat",): ( + "Mach Virtual Memory Statistics: (page size of 16384 bytes)\n" + "Pages free: 1000000.\n" + "Pages inactive: 1000000.\n" + "Pages speculative: 1000000.\n" + ), + ("ps", "-axo", "pid=,ppid=,command="): "", + } + return commands[args] + + runner = dict(DS4_RUNNER_ATTESTATION) + runner["runner_executable"] = str(runner_executable.resolve()) + runner["runner_executable_sha256"] = trace.sha256_file(runner_executable) + runner["runner_script"] = str(runner_script.resolve()) + runner["runner_script_sha256"] = trace.sha256_file(runner_script) + runner["exporter_path"] = str(exporter.resolve()) + runner["exporter_sha256"] = trace.sha256_file(exporter) + runner["checkout_path"] = str((root / "ds4").resolve()) + with mock.patch.dict(preflight.os.environ, {"TMPDIR": str(root / "tmp")}, clear=True): + result = preflight.run_oracle_preflight( + model=model, + prompt=prompt, + output=root / "output", + repo=root / "repo", + checkout=root / "ds4", + busy_patterns=[], + accelerator=dict(METAL_ACCELERATOR_ATTESTATION), + runner=runner, + disk_info=disk_info, + command_text=command_text, + system="Darwin", + machine="arm64", + ) + self.assertEqual(result["runtime_kind"], "apple-metal") + self.assertEqual(result["storage"]["model"]["storage_kind"], "darwin-local-solid-state") + self.assertEqual(result["host"]["memory_bytes"], 256 * 1024 * 1024 * 1024) + + for mutation, message in ( + ({"Internal": False}, "internal non-rotational"), + ({"SolidState": False}, "internal non-rotational"), + ({"VolumeNetwork": True}, "local storage"), + ({"DiskImage": True}, "local storage"), + ({"BusProtocol": "Network"}, "bus protocol")): + def invalid_info(path: Path, mutation: dict[str, object] = mutation) -> dict[str, object]: + result = disk_info(path) + result.update(mutation) + return result + + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.darwin_storage_attestation( + model, + "model", + disk_info=invalid_info, + ) + + with self.assertRaisesRegex(preflight.PreflightError, "macOS on arm64"): + preflight.darwin_host_and_memory_audit( + command_text=command_text, + system="Linux", + machine="x86_64", + ) + + def test_darwin_storage_queries_the_containing_mount(self) -> None: + path = Path("/Users/test/model.gguf") + disk_info = { + "DeviceIdentifier": "disk3s5", + "ParentWholeDisk": "disk3", + "BusProtocol": "Apple Fabric", + "Internal": True, + "SolidState": True, + } + + def check_output(command, **_kwargs): + if command == ["df", "-P", str(path)]: + return ( + "Filesystem 512-blocks Used Available Capacity Mounted on\n" + "/dev/disk3s5 100 10 90 10% /System/Volumes/Data\n" + ) + self.assertEqual(command, ["diskutil", "info", "-plist", "/System/Volumes/Data"]) + return preflight.plistlib.dumps(disk_info) + + with mock.patch.object(preflight.subprocess, "check_output", side_effect=check_output): + result = preflight._diskutil_info(path) + self.assertEqual(result["_dsv41_mount_point"], "/System/Volumes/Data") + self.assertEqual({key: value for key, value in result.items() if not key.startswith("_")}, disk_info) def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: with mock.patch.object( @@ -808,13 +1176,43 @@ def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: return_value=storage_record("/home/test")): with mock.patch.dict(preflight.os.environ, {"HIP_LAUNCH_BLOCKING": "1"}, clear=True): with self.assertRaisesRegex(preflight.PreflightError, "TMPDIR is required"): - preflight.run_preflight( + preflight.run_strix_preflight( model=Path("/home/model.gguf"), prompt=Path("/home/prompt.txt"), output=Path("/home/trace"), repo=Path("/home/repo"), busy_patterns=[], ) + with tempfile.TemporaryDirectory() as temp: + missing = Path(temp) / "missing" + with mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(missing)}, + clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "existing writable directory"): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + actual = Path(temp) / "actual" + actual.mkdir() + link = Path(temp) / "link" + link.symlink_to(actual, target_is_directory=True) + with mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(link)}, + clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "must not be a symlink"): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) def test_watchdog_lease_rejects_arbitrary_heartbeat_process(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -870,20 +1268,12 @@ def stat(pid: int, parent: int, start: int) -> str: "sample": {}, }), encoding="ascii") child_argv = ["python3", "run_matrix.py"] + watchdog_events = json.loads(json.dumps(WATCHDOG_EVENTS)) + watchdog_events[1]["child_pid"] = child_pid + watchdog_events[1]["process_group_id"] = child_pid + watchdog_events[1]["command"] = child_argv audit.write_text( - json.dumps({ - "event": "preflight", - "soft_bytes": preflight.SOFT_MEMORY_LIMIT, - "emergency_bytes": preflight.WATCHDOG_EMERGENCY_LIMIT, - "strict_ceiling_bytes": preflight.STRICT_MEMORY_LIMIT, - "swap_entries": 0, - }) + "\n" + - json.dumps({ - "event": "child_started", - "child_pid": child_pid, - "process_group_id": child_pid, - "command": child_argv, - }) + "\n", + "".join(json.dumps(event) + "\n" for event in watchdog_events), encoding="ascii", ) lease_record = { @@ -1136,6 +1526,241 @@ def test_rejects_unattested_accelerator_identity(self) -> None: with self.assertRaisesRegex(trace.TraceError, message): trace.TraceBundle(root) + def test_accepts_truthful_metal_vs_strix_bundles(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "ds4" + right = Path(temp) / "llama" + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: + add_required_events(writer) + left_bundle = trace.TraceBundle(left) + right_bundle = trace.TraceBundle(right) + self.assertNotEqual( + left_bundle.manifest["accelerator"]["architecture"], + right_bundle.manifest["accelerator"]["architecture"], + ) + result = trace.report(left_bundle, right_bundle) + self.assertEqual(result["status"], "TARGET PASS") + + def test_rejects_cross_runtime_attestation_substitution(self) -> None: + for runtime, accelerator, message in ( + ("ds4", ACCELERATOR_ATTESTATION, "ds4 accelerator attestation fields"), + ("llama.cpp", METAL_ACCELERATOR_ATTESTATION, "llama.cpp accelerator attestation fields")): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest(runtime) + trace_manifest["accelerator"] = dict(accelerator) + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + for runtime, records, replacement, message in ( + ( + "ds4", + DS4_AUDIT_RECORDS, + storage_record("/mnt/models/model.gguf"), + "ds4 storage attestation fields", + ), + ( + "llama.cpp", + AUDIT_RECORDS, + metal_storage_record("/Users/oracle/model.gguf"), + "llama.cpp storage attestation fields", + )): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest(runtime)) as writer: + add_required_events(writer) + record = json.loads(json.dumps(records["memory"])) + record["storage"]["model"] = replacement + replace_audit_record(root, "pre", "memory", record) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_rejects_ds4_accelerator_audit_removal(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["memory"])) + del record["accelerator"] + replace_audit_record(root, phase, "memory", record) + with self.assertRaisesRegex(trace.TraceError, "missing accelerator"): + trace.TraceBundle(root) + + def test_rejects_cross_runtime_host_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("llama.cpp") + trace_manifest["host"] = dict(DS4_HOST_ATTESTATION) + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "unexpected host"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["host"]["runtime_kind"] = "strix-rocm" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "host runtime_kind mismatch"): + trace.TraceBundle(root) + + def test_rejects_storage_audit_path_substitution(self) -> None: + for runtime, records, different in ( + ("llama.cpp", AUDIT_RECORDS, "/mnt/models/different.gguf"), + ("ds4", DS4_AUDIT_RECORDS, "/Users/oracle/different.gguf")): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest(runtime)) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(records["memory"])) + record["storage"]["model"]["resolved_path"] = different + record["storage"]["model"]["existing_path"] = different + replace_audit_record(root, phase, "memory", record) + with self.assertRaisesRegex(trace.TraceError, "model path differs from the manifest"): + trace.TraceBundle(root) + + def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + data = manifest_path.read_text(encoding="ascii") + data = data.replace( + '"runtime_kind":"apple-metal"', + '"runtime_kind":"apple-metal","runtime_kind":"apple-metal"', + 1, + ) + manifest_path.write_text(data, encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "duplicate JSON key"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["accelerator"]["runtime_kind"] = "unknown" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "runtime_kind mismatch"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + del trace_manifest["accelerator"]["runtime_kind"] + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "fields are invalid"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["unknown_top_level"] = True + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "unexpected unknown_top_level"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["config"]["unvalidated_mode"] = "unsafe" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "unexpected unvalidated_mode"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["environment"]["system_info"] = "Linux test system" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "environment is not macOS"): + trace.TraceBundle(root) + + def test_rejects_boolean_accelerator_identities(self) -> None: + for runtime, field in ( + ("llama.cpp", "gpu_id"), + ("ds4", "metal_registry_id"), + ("ds4", "recommended_max_working_set_bytes")): + with self.subTest(runtime=runtime, field=field), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest(runtime) + trace_manifest["accelerator"][field] = True + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "invalid"): + trace.TraceBundle(root) + + def test_rejects_unknown_watchdog_event_fields(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + events = json.loads(json.dumps(WATCHDOG_EVENTS)) + events[0]["unknown"] = True + data = "".join( + json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + for event in events + ).encode("ascii") + digest = trace.sha256_bytes(data) + for phase in ("pre", "post"): + jsonl = root / "audits" / phase / f"{digest}.jsonl" + jsonl.write_bytes(data) + record = json.loads(json.dumps(AUDIT_RECORDS["watchdog"])) + record["data"]["audit"]["path"] = f"audits/{phase}/{digest}.jsonl" + record["data"]["audit"]["sha256"] = digest + record["data"]["audit"]["event_count"] = len(events) + replace_audit_record(root, phase, "watchdog", record) + with self.assertRaisesRegex(trace.TraceError, "unexpected unknown"): + trace.TraceBundle(root) + + def test_ds4_runner_path_must_match_attested_storage(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["runner"])) + record["data"]["runner_script"] = "/Users/oracle/other/run_ds4.py" + replace_audit_record(root, phase, "runner", record) + with self.assertRaisesRegex(trace.TraceError, "runner_script mismatch"): + trace.TraceBundle(root) + + def test_unapproved_ds4_exporter_is_not_executed(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + exporter = root / "exporter" + exporter.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") + exporter.chmod(0o755) + argv = [ + "run_ds4.py", + "--repo", str(root), + "--model", str(root / "model.gguf"), + "--prompt", str(root / "prompt.txt"), + "--output", str(root / "output"), + "--exporter", str(exporter), + "--exporter-sha256", trace.sha256_file(exporter), + "--corpus-name", "correctness-prose.txt", + "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], + "--prompt-provenance", str(root / "prompt.json"), + "--preflight-only", + ] + with mock.patch.object(sys, "argv", argv), mock.patch.object( + sys, "stderr", io.StringIO()), mock.patch.object( + run_ds4, "query_accelerator_attestation") as query: + self.assertEqual(run_ds4.main(), 1) + query.assert_not_called() + def test_rejects_wrong_component_schema_and_same_bundle_compare(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 8f6044bd9a7a..766d5b13c04b 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -4,13 +4,15 @@ This directory defines version 2 of the cross-runtime trace format used by issue Each trace is a directory: -- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, and content-addressed memory/swap/watchdog audit references. +- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, runtime-specific host evidence, exact execution paths, and content-addressed audit references. - `events.jsonl` is an ordered stream of content-addressed event records. - `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. - `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. - `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. -The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. Every bundle also carries an exact accelerator attestation. The selected backend device must map through its PCI identity and Linux KFD topology to `gfx_target_version=110501` (`gfx1151`); device labels or environment strings are not accepted as architecture evidence. +The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. + +Every bundle carries one strict runtime-discriminated accelerator attestation. A llama.cpp candidate uses the `strix-rocm` kind: the selected backend device must map through its PCI identity and Linux KFD topology to `gfx_target_version=110501` (`gfx1151`). A ds4 oracle uses the `apple-metal` kind: the selected Metal device records its registry ID, reported architecture, unified-memory property, and recommended working-set size. Missing kinds, unknown kinds, cross-kind fields, duplicate JSON keys, and mixed evidence fail closed. Cross-runtime comparison does not require the two physical accelerators or PCI identities to match; each runtime proves its own execution environment, while the model, prompt, inference semantics, and complete output artifacts remain exact comparison inputs. Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, layer-0/1 raw attention-source rows as i32 `[128, token_count]`, compressed attention-source IDs as nonempty rank-2 i32 with width at most 512, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Raw attention rows use physical ring IDs `0..127`, visible current-ubatch IDs `128..128+token_index`, and unavailable sentinel `128+token_count`; layers 0 and 1 must be byte-identical for each execution step. Original expert IDs must be within `0..383`. @@ -71,17 +73,17 @@ build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o CPY Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly, invokes the exact exporter for a pre-allocation device attestation, and rejects the run unless the backend PCI identity maps to exactly one KFD node reporting `gfx1151`. The native exporter repeats the query before model allocation and verifies that the loaded model still uses the same device. -Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the complete matrix process group. The wrappers fail closed if this variable is absent or different, and every embedded memory, swap, and watchdog audit records it. Keep the same inherited value for ds4 and llama.cpp. +Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the complete Strix matrix process group. The llama.cpp wrapper fails closed if this variable is absent or different, and every embedded Strix memory, swap, and watchdog audit records it. This Linux/ROCm setting is not an Apple Metal oracle requirement. -## Strix execution gate +## Strix candidate execution gate -`run_ds4.py` verifies that the pinned ds4 checkout has no tracked or untracked changes and refuses model execution when swap is enabled, the canonical watchdog lease, heartbeat, or JSONL audit is missing or stale, another unrelated matching DS4 workload is active, or any model/prompt/trace path resolves under `/mnt/bigspace`. +`run_llama.py` refuses model execution when swap is enabled, the canonical watchdog lease, heartbeat, or JSONL audit is missing or stale, another unrelated matching model workload is active, or any model/prompt/trace path fails the storage gate. The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd0747962b`, with `scripts/strix_memory_watchdog.py` SHA-256 `d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f`. Both Python validators and the native exporter reject every other revision or script hash. The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. -Use one empty directory on verified non-rotational NVMe for every input and output. The Python launchers and both native tools resolve symlinks and the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, and `/mnt/bigspace` fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory and has no `/tmp` fallback. These metadata commands do not execute the model: +Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory, must be an existing writable non-symlink directory on verified NVMe, and has no `/tmp` fallback. These metadata commands do not execute the model: ```sh MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf @@ -107,7 +109,7 @@ git -C /home/papa/src/ds4-v41 rev-parse HEAD test "$(awk 'NR > 1 { count++ } END { print count + 0 }' /proc/swaps)" = 0 ``` -The expected model digest is `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`, the ds4 status output is empty, the ds4 revision is `bd66c402070042bf0a79ad6ece8242de4c93680c`, both selected block devices report `ROTA=0`, and `/proc/swaps` has zero entries. The observed planning snapshot had 76366495744 bytes free on `/mnt/models` and 679635001344 bytes free under `/home`; recheck before every run. Keep the unchanged 365713686528-byte GGUF in place on `/mnt/models`. Do not copy the model or place builds, logs, traces, audit files, or temporary files there. Put all of those under `/home`, and never use `/mnt/bigspace`. +The expected model digest is `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`, both selected block devices report `ROTA=0`, and `/proc/swaps` has zero entries. The observed planning snapshot had 76366495744 bytes free on `/mnt/models` and 679635001344 bytes free under `/home`; recheck before every run. Keep the unchanged 365713686528-byte GGUF in place on `/mnt/models`. Do not copy the model or place builds, logs, traces, audit files, or temporary files there. Put all of those under `/home`, and never use `/mnt/bigspace`. The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision and its build SHA-256 matches the executed file. @@ -115,31 +117,34 @@ The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the no Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision, executable hash, accelerator identity, or loaded model device does not match that attestation. -`run_matrix.py` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, runs ds4 and llama.cpp with matched context/decode settings, compares each bundle immediately, and stops at the first divergence. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. +`run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, and captures the llama.cpp side. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. + +## Apple Metal oracle execution gate + +The external ds4 exporter is not present in the pinned canonical checkout. It remains blocked until a separately built executable is reviewed and its exact SHA-256 is added to the otherwise empty `APPROVED_EXPORTERS` map. Approval is checked before the exporter can run, including device-only preflight. `run_ds4.py` requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state local storage. Network volumes, disk images, external/non-internal devices, non-solid-state media, incomplete device identity, and lexical or resolved forbidden paths fail closed. -The external ds4 exporter is not present in the pinned `/home/papa/src/ds4-v41` checkout. It remains a blocker until a separately built executable is provided and attested. `run_ds4.py` currently has no approved exporter digest and fails closed before inference. After the exporter is implemented and reviewed on an authorized oracle host, add its exact executable SHA-256 and pinned ds4 revision to `APPROVED_EXPORTERS` in `run_ds4.py`; a caller-provided digest alone is not sufficient oracle provenance. Its bundle must include the same KFD-derived `gfx1151` accelerator identity as the llama.cpp trace. The exporter must accept the interface used by `run_ds4.py`: +The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. + +After the exporter is independently reviewed on an authorized 128 GiB or larger Apple oracle host, add its exact executable SHA-256 and pinned ds4 revision to `APPROVED_EXPORTERS` in `run_ds4.py`; a caller-provided digest alone is not sufficient oracle provenance. The exporter must answer `--dsv41-attest-device Metal0` without loading the model and emit the strict `apple-metal` attestation. Its trace command interface is: The unpublished `ds4gguf` documentation revision `e13893ffcb33e90c8852929303e188102df7a8f5` is provenance only. It is not an executable dependency, exporter approval, or fixture source. Executable tests and fixtures stay in this `strix-llama.cpp` stack. ```text ---model PATH --prompt-file PATH --output PATH --context N --decode-steps N --prefill-chunk 32 ---memory-audit PATH --swap-audit PATH --watchdog-audit PATH +--model PATH --prompt-file PATH --output PATH --context N --decode-steps N --prefill-chunk 32 --device Metal0 ``` -It must emit a complete valid `dsv41-trace` bundle, report ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`, and put its own executable SHA-256 in `manifest.json`. +It must emit a complete valid `dsv41-trace` bundle, report ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`, put its own executable SHA-256 in `manifest.json`, and report the same selected Metal device before and after execution. `run_ds4.py` embeds the platform-native memory, swap, runner, accelerator, host, storage, and exact-path evidence. -After the exporter exists, set the immutable identities and run the first matrix under the watchdog: +Capture the first llama.cpp matrix under the watchdog: ```sh REPO=/home/papa/src/strix-llama-integration MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf RUN_ROOT=/home/papa/dsv41-correctness CASE_ROOT="$RUN_ROOT/c32768" -DS4_EXPORTER=/home/papa/bin/dsv41-trace-exporter CANDIDATE_REV="$(git -C "$REPO" rev-parse HEAD)" BASE_REV= DIFF_SHA256="$(git -C "$REPO" diff --binary --no-ext-diff "$BASE_REV" "$CANDIDATE_REV" -- | sha256sum | awk '{print $1}')" -DS4_EXPORTER_SHA256="$(sha256sum "$DS4_EXPORTER" | awk '{print $1}')" mkdir -p "$CASE_ROOT/watchdog" cd "$REPO" @@ -164,10 +169,7 @@ HIP_LAUNCH_BLOCKING=1 python3 scripts/strix_memory_watchdog.py \ --candidate-revision "$CANDIDATE_REV" \ --base-revision "$BASE_REV" \ --candidate-diff-sha256 "$DIFF_SHA256" \ - --ds4-runner "$REPO/tools/deepseek-v41-trace/run_ds4.py" \ - --ds4-checkout /home/papa/src/ds4-v41 \ - --ds4-exporter "$DS4_EXPORTER" \ - --ds4-exporter-sha256 "$DS4_EXPORTER_SHA256" \ + --llama-only \ --contexts 32768 \ --ubatches 32 \ --batch 2048 \ @@ -184,6 +186,29 @@ CASE_ROOT="$RUN_ROOT/c98304" # then use --contexts 98304 CASE_ROOT="$RUN_ROOT/c131072" # then use --contexts 131072 ``` +On the separately authorized Apple oracle, place the unchanged GGUF, exact prompt, prompt-provenance record, harness checkout, pinned ds4 checkout, exporter, trace output, and `TMPDIR` on internal solid-state storage. Then run one case at a time: + +```sh +export TMPDIR=/Users/oracle/dsv41/tmp +python3 tools/deepseek-v41-trace/run_ds4.py \ + --repo /Users/oracle/src/strix-llama.cpp \ + --checkout /Users/oracle/src/ds4-v41 \ + --exporter /Users/oracle/bin/dsv41-trace-exporter \ + --exporter-sha256 \ + --model /Users/oracle/models/DeepSeek-V4.1-Flash-Q2.gguf \ + --prompt /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt \ + --prompt-provenance /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt.provenance.json \ + --output /Users/oracle/dsv41/traces/correctness-prose-c32768-ub32 \ + --corpus-name correctness-prose.txt \ + --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ + --context 32768 \ + --decode-steps 8 \ + --prefill-chunk 32 \ + --device Metal0 +``` + +Compare the completed bundle with the matching Strix bundle using `trace_format.py compare`. Repeat for all four corpora before expanding the context matrix. + ## Strix bring-up evidence lane The 128 GiB or larger Apple Metal run remains an external verification gate for the pinned ds4 cross-runtime oracle. A Strix bring-up can complete first, but it must report `BRINGUP PASS`, never `TARGET PASS`. It does not replace complete byte-identical ds4 logits. @@ -215,7 +240,7 @@ python3 tools/deepseek-v41-trace/verify_ds4_anchors.py \ --checkout /home/papa/src/ds4-v41 ``` -`run_matrix.py --llama-only` captures all four repository corpora without claiming cross-runtime success. Use the matrix command above, add `--llama-only`, and omit `--ds4-runner`, `--ds4-checkout`, `--ds4-exporter`, and `--ds4-exporter-sha256`. Run the final integration build twice with separate output directories, then run an equivalently instrumented immutable base build once. Keep every run under its own canonical watchdog invocation and use the exact ubatch/cache/ROCm arguments above. +`run_matrix.py --llama-only` captures all four repository corpora without claiming cross-runtime success. Run the final integration build twice with separate output directories, then run an equivalently instrumented immutable base build once. Keep every run under its own canonical watchdog invocation and use the exact ubatch/cache/ROCm arguments above. For each case (`correctness-prose-c32768-ub32`, `correctness-code-c32768-ub32`, `correctness-structured-c32768-ub32`, and `correctness-numeric-c32768-ub32`), require both comparisons: diff --git a/tools/deepseek-v41-trace/host-attestation.h b/tools/deepseek-v41-trace/host-attestation.h index 5476f3369549..a55dd3aa903e 100644 --- a/tools/deepseek-v41-trace/host-attestation.h +++ b/tools/deepseek-v41-trace/host-attestation.h @@ -14,6 +14,12 @@ #include #include +#if defined(_WIN32) +#include +#else +#include +#endif + namespace dsv41 { namespace fs = std::filesystem; @@ -98,16 +104,36 @@ static inline fs::path existing_ancestor(const fs::path & path) { return fs::canonical(current); } +static inline void require_usable_directory(const fs::path & path, const char * label) { + if (fs::is_symlink(path)) { + throw std::runtime_error(std::string(label) + " must not be a symlink"); + } + if (!fs::is_directory(path)) { + throw std::runtime_error(std::string(label) + " must be an existing directory"); + } +#if defined(_WIN32) + if (_access(path.string().c_str(), 6) != 0) { +#else + if (access(path.string().c_str(), W_OK | X_OK) != 0) { +#endif + throw std::runtime_error(std::string(label) + " must be writable and searchable"); + } +} + static inline storage_attestation require_nvme_path( const fs::path & path, const char * label, const fs::path & mountinfo_path = "/proc/self/mountinfo", const fs::path & sys_dev_block_root = "/sys/dev/block", - const fs::path & sys_class_block_root = "/sys/class/block") { + const fs::path & sys_class_block_root = "/sys/class/block", + const fs::path & forbidden_root = "/mnt/bigspace") { const fs::path absolute = fs::absolute(path).lexically_normal(); + const fs::path forbidden = fs::absolute(forbidden_root).lexically_normal(); + if (path_is_within(absolute, forbidden)) { + throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); + } const fs::path resolved = fs::weakly_canonical(absolute); const fs::path existing = existing_ancestor(resolved); - const fs::path forbidden = "/mnt/bigspace"; if (path_is_within(resolved, forbidden)) { throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); } diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 32739dd72682..5ad7144de315 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -623,7 +623,10 @@ static std::vector command_line(int argc, char ** argv) { static json accelerator_json(const dsv41::accelerator_attestation & accelerator) { return { {"format", "dsv41-accelerator-attestation"}, - {"version", 1}, + {"version", 2}, + {"runtime_kind", "strix-rocm"}, + {"platform", "linux"}, + {"backend", "ROCm"}, {"backend_device", accelerator.backend_device}, {"backend_description", accelerator.backend_description}, {"pci_device_id", accelerator.pci_device_id}, @@ -635,6 +638,26 @@ static json accelerator_json(const dsv41::accelerator_attestation & accelerator) }; } +static json storage_json(const dsv41::storage_attestation & storage) { + return { + {"format", "dsv41-storage-attestation"}, + {"version", 2}, + {"runtime_kind", "strix-rocm"}, + {"platform", "linux"}, + {"storage_kind", "linux-nvme"}, + {"resolved_path", storage.resolved_path.string()}, + {"existing_path", storage.existing_path.string()}, + {"mount_point", storage.mount_point}, + {"filesystem_type", storage.filesystem_type}, + {"mount_source", storage.mount_source}, + {"device_number", storage.device_number}, + {"block_device_path", storage.block_device_path.string()}, + {"nvme_device", storage.nvme_device}, + {"rotational", false}, + {"source", "linux-mountinfo-sysfs"}, + }; +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { @@ -673,6 +696,10 @@ int main(int argc, char ** argv) { dsv41::require_nvme_path(params.prompt_file, "prompt"); const dsv41::storage_attestation output_storage = dsv41::require_nvme_path(params.out_file, "trace output"); + const fs::path temporary_directory = required_environment("TMPDIR"); + dsv41::require_usable_directory(temporary_directory, "TMPDIR"); + const dsv41::storage_attestation temporary_storage = + dsv41::require_nvme_path(temporary_directory, "temporary directory"); if (required_environment("HIP_LAUNCH_BLOCKING") != "1") { throw std::runtime_error("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs"); } @@ -685,6 +712,13 @@ int main(int argc, char ** argv) { if (memory_audit.value("accelerator", json::object()) != accelerator_json(configured_accelerator)) { throw std::runtime_error("preflight accelerator audit does not match the selected execution device"); } + const json audited_storage = memory_audit.value("storage", json::object()); + if (audited_storage.value("model", json::object()) != storage_json(model_storage) || + audited_storage.value("prompt", json::object()) != storage_json(prompt_storage) || + audited_storage.value("output", json::object()) != storage_json(output_storage) || + audited_storage.value("temporary_directory", json::object()) != storage_json(temporary_storage)) { + throw std::runtime_error("preflight storage audit does not match the selected execution paths"); + } const json swap_audit = audit_reference("DSV41_TRACE_SWAP_AUDIT", "swap"); const json watchdog_audit = audit_reference("DSV41_TRACE_WATCHDOG_AUDIT", "watchdog"); @@ -762,6 +796,13 @@ int main(int argc, char ** argv) { {"sha256", sha256_data(prompt_bytes.data(), prompt_bytes.size())}, }}, {"accelerator", accelerator_json(accelerator)}, + {"paths", { + {"model", model_path.string()}, + {"prompt", prompt_path.string()}, + {"output", output_path.string()}, + {"repository", audited_storage["repository"].value("resolved_path", "")}, + {"temporary_directory", temporary_storage.resolved_path.string()}, + }}, {"config", { {"context", llama_n_ctx(ctx)}, {"batch", params.n_batch}, diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index ad717bc20037..95c846a4aba8 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -4,13 +4,18 @@ import hashlib import importlib.util import os +import platform +import plistlib import re +import subprocess import sys import time from datetime import datetime from pathlib import Path from typing import Callable +from trace_format import TraceError, validate_watchdog_event + FORBIDDEN_ROOT = Path("/mnt/bigspace") SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 WATCHDOG_EMERGENCY_LIMIT = 118 * 1024 * 1024 * 1024 @@ -34,6 +39,21 @@ class PreflightError(RuntimeError): pass +def strict_json_loads(data: str) -> object: + def reject_duplicates(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise PreflightError(f"duplicate JSON key: {key}") + result[key] = value + return result + + try: + return json.loads(data, object_pairs_hook=reject_duplicates) + except json.JSONDecodeError as error: + raise PreflightError(f"invalid JSON: {error}") from error + + def resolved(path: Path) -> Path: return path.expanduser().resolve() @@ -61,10 +81,18 @@ def storage_attestation( *, mountinfo_path: Path = Path("/proc/self/mountinfo"), sys_dev_block_root: Path = Path("/sys/dev/block"), - sys_class_block_root: Path = Path("/sys/class/block")) -> dict[str, object]: - path = resolved(path) + sys_class_block_root: Path = Path("/sys/class/block"), + forbidden_root: Path = FORBIDDEN_ROOT) -> dict[str, object]: + lexical_path = path.expanduser().absolute() + try: + lexical_path.relative_to(forbidden_root) + except ValueError: + pass + else: + raise PreflightError(f"{label} must not use /mnt/bigspace: {lexical_path}") + path = resolved(lexical_path) try: - path.relative_to(FORBIDDEN_ROOT) + path.relative_to(forbidden_root) except ValueError: pass else: @@ -135,6 +163,11 @@ def storage_attestation( if nvme_device is None: raise PreflightError(f"{label} must use an NVMe block device: {block_device}") return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "storage_kind": "linux-nvme", "resolved_path": str(path), "existing_path": str(existing), "mount_point": str(mount_point), @@ -144,6 +177,7 @@ def storage_attestation( "block_device_path": str(block_device), "nvme_device": nvme_device, "rotational": False, + "source": "linux-mountinfo-sysfs", } @@ -164,6 +198,101 @@ def require_nvme_path( return _attested_resolved_path(attestation, label) +def _diskutil_info(path: Path) -> dict[str, object]: + try: + df = subprocess.check_output( + ["df", "-P", str(path)], + text=True, + stderr=subprocess.STDOUT, + ).splitlines() + if len(df) < 2: + raise PreflightError(f"df cannot resolve a mounted volume for {path}") + fields = df[-1].split(maxsplit=5) + if len(fields) != 6 or not fields[5].startswith("/"): + raise PreflightError(f"df returned invalid mount evidence for {path}") + mount_point = fields[5] + data = subprocess.check_output( + ["diskutil", "info", "-plist", mount_point], + stderr=subprocess.STDOUT, + ) + record = plistlib.loads(data) + except (OSError, subprocess.CalledProcessError, plistlib.InvalidFileException) as error: + raise PreflightError(f"diskutil cannot attest storage for {path}: {error}") from error + if not isinstance(record, dict): + raise PreflightError(f"diskutil returned invalid storage evidence for {path}") + record["_dsv41_mount_point"] = mount_point + return record + + +def darwin_storage_attestation( + path: Path, + label: str, + *, + disk_info: Callable[[Path], dict[str, object]] = _diskutil_info, + forbidden_root: Path = FORBIDDEN_ROOT) -> dict[str, object]: + lexical_path = path.expanduser().absolute() + try: + lexical_path.relative_to(forbidden_root) + except ValueError: + pass + else: + raise PreflightError(f"{label} must not use /mnt/bigspace: {lexical_path}") + path = resolved(lexical_path) + try: + path.relative_to(forbidden_root) + except ValueError: + pass + else: + raise PreflightError(f"{label} must not use /mnt/bigspace: {path}") + existing = _existing_ancestor(path) + record = dict(disk_info(existing)) + mount_point = record.pop("_dsv41_mount_point", record.get("MountPoint")) + filesystem_type = record.get("FilesystemType") + device_identifier = record.get("DeviceIdentifier") + parent_whole_disk = record.get("ParentWholeDisk") + bus_protocol = record.get("BusProtocol") + if record.get("Internal") is not True or record.get("SolidState") is not True: + raise PreflightError(f"{label} must use internal non-rotational storage: {path}") + if record.get("VolumeNetwork") is True or record.get("DiskImage") is True: + raise PreflightError(f"{label} must use local storage: {path}") + for name, value in ( + ("mount point", mount_point), + ("filesystem type", filesystem_type), + ("device identifier", device_identifier), + ("parent whole disk", parent_whole_disk), + ("bus protocol", bus_protocol)): + if not isinstance(value, str) or not value: + raise PreflightError(f"{label} {name} cannot be resolved: {path}") + if not mount_point.startswith("/"): + raise PreflightError(f"{label} mount point is invalid: {mount_point}") + if bus_protocol.lower() in {"network", "virtual", "disk image"}: + raise PreflightError(f"{label} bus protocol is not local: {bus_protocol}") + try: + filesystem_device = os.stat(existing).st_dev + if filesystem_device != os.stat(mount_point).st_dev: + raise PreflightError(f"{label} filesystem identity differs from its attested mount: {path}") + except OSError as error: + raise PreflightError(f"{label} filesystem identity cannot be read: {error}") from error + return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "storage_kind": "darwin-local-solid-state", + "resolved_path": str(path), + "existing_path": str(existing), + "mount_point": mount_point, + "filesystem_type": filesystem_type, + "device_identifier": device_identifier, + "parent_whole_disk": parent_whole_disk, + "bus_protocol": bus_protocol, + "filesystem_device": filesystem_device, + "internal": True, + "solid_state": True, + "source": "diskutil-info-plist", + } + + def _attested_resolved_path(attestation: dict[str, object], label: str) -> Path: resolved_path = attestation["resolved_path"] if not isinstance(resolved_path, str): @@ -175,7 +304,7 @@ def safe_trace_path(root: Path, relative: Path | str) -> Path: root = root.expanduser().absolute() if root.is_symlink(): raise PreflightError("trace output root must not be a symlink") - root = require_nvme_path(root, "trace output") + root = root.resolve() relative = Path(relative) if relative.is_absolute() or ".." in relative.parts: raise PreflightError(f"trace output path is outside the bundle: {relative}") @@ -188,7 +317,6 @@ def safe_trace_path(root: Path, relative: Path | str) -> Path: candidate.resolve().relative_to(root) except ValueError as error: raise PreflightError(f"trace output path is outside the bundle: {relative}") from error - require_nvme_path(candidate, "trace output") return candidate @@ -236,6 +364,86 @@ def memory_audit() -> dict[str, int]: return result +def _command_text(*args: str) -> str: + try: + return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"{' '.join(args)} failed: {error}") from error + + +def darwin_host_and_memory_audit( + *, + command_text: Callable[..., str] = _command_text, + system: str | None = None, + machine: str | None = None) -> tuple[dict[str, object], dict[str, int]]: + platform_name = platform.system() if system is None else system + machine_name = platform.machine() if machine is None else machine + if platform_name != "Darwin" or machine_name != "arm64": + raise PreflightError("ds4 oracle execution requires macOS on arm64") + try: + total = int(command_text("sysctl", "-n", "hw.memsize")) + except ValueError as error: + raise PreflightError("Darwin memory size is invalid") from error + if total < 128 * 1024 * 1024 * 1024: + raise PreflightError("ds4 oracle host must have at least 128 GiB of memory") + vm_stat = command_text("vm_stat") + page_size_match = re.search(r"page size of ([0-9]+) bytes", vm_stat) + if page_size_match is None: + raise PreflightError("vm_stat page size is missing") + page_size = int(page_size_match.group(1)) + pages = {} + for name, value in re.findall(r"^([^:]+):\s+([0-9]+)\.$", vm_stat, flags=re.MULTILINE): + pages[name] = int(value) + available_pages = sum(pages.get(name, 0) for name in ( + "Pages free", + "Pages inactive", + "Pages speculative", + )) + available = available_pages * page_size + if available <= 0 or available > total: + raise PreflightError("Darwin available memory evidence is invalid") + host = { + "format": "dsv41-host-attestation", + "version": 1, + "runtime_kind": "apple-metal", + "platform": "macos", + "machine": "arm64", + "hardware_model": command_text("sysctl", "-n", "hw.model"), + "os_version": command_text("sysctl", "-n", "kern.osproductversion"), + "memory_bytes": total, + "source": "darwin-sysctl", + } + if not host["hardware_model"] or not host["os_version"]: + raise PreflightError("Darwin host identity is incomplete") + return host, { + "mem_total_bytes": total, + "mem_available_bytes": available, + "mem_used_bytes": total - available, + } + + +def darwin_swap_audit( + *, + command_text: Callable[..., str] = _command_text) -> dict[str, object]: + value = command_text("sysctl", "-n", "vm.swapusage") + match = re.fullmatch( + r"total = ([0-9]+(?:\.[0-9]+)?)M\s+used = ([0-9]+(?:\.[0-9]+)?)M\s+" + r"free = ([0-9]+(?:\.[0-9]+)?)M(?:\s+\(encrypted\))?", + value, + ) + if match is None: + raise PreflightError("Darwin swap evidence is invalid") + total, used, free = (int(float(item) * 1024 * 1024) for item in match.groups()) + if total != 0 or used != 0 or free != 0: + raise PreflightError("swap is enabled; model execution is blocked") + return { + "source": "darwin-sysctl-vm.swapusage", + "total_bytes": total, + "used_bytes": used, + "free_bytes": free, + } + + def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() @@ -285,7 +493,7 @@ def read_heartbeat( now: int | None = None, monotonic_ns: Callable[[], int] = time.monotonic_ns) -> int: try: - record = json.loads(path.read_text(encoding="ascii")) + record = strict_json_loads(path.read_text(encoding="ascii")) updated = datetime.fromisoformat(str(record["updated_at"]).replace("Z", "+00:00")) heartbeat = int(updated.timestamp()) except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: @@ -294,9 +502,9 @@ def read_heartbeat( raise PreflightError("watchdog heartbeat format is invalid") if record.get("lease_id") != lease_id or record.get("state") != "active": raise PreflightError("watchdog heartbeat lease identity or state is invalid") - if not isinstance(record.get("sequence"), int) or record["sequence"] < 0: + if type(record.get("sequence")) is not int or record["sequence"] < 0: raise PreflightError("watchdog heartbeat sequence is invalid") - if not isinstance(record.get("updated_monotonic_ns"), int) or record["updated_monotonic_ns"] <= 0: + if type(record.get("updated_monotonic_ns")) is not int or record["updated_monotonic_ns"] <= 0: raise PreflightError("watchdog heartbeat monotonic timestamp is invalid") if record.get("watchdog_pid") != watchdog_pid or ( record.get("watchdog_start_time_ticks") != watchdog_start_time_ticks): @@ -322,7 +530,7 @@ def _read_json_with_retry( last_error: Exception | None = None while True: try: - record = json.loads(path.read_text(encoding="ascii")) + record = strict_json_loads(path.read_text(encoding="ascii")) if not isinstance(record, dict): raise ValueError("record is not an object") return record @@ -341,11 +549,9 @@ def _read_watchdog_events(path: Path) -> list[dict[str, object]]: events = [] for line_number, line in enumerate(lines, start=1): try: - event = json.loads(line) - except json.JSONDecodeError as error: + event = validate_watchdog_event(strict_json_loads(line)) + except (PreflightError, TraceError) as error: raise PreflightError(f"watchdog audit line {line_number} is invalid: {error}") from error - if not isinstance(event, dict) or not isinstance(event.get("event"), str): - raise PreflightError(f"watchdog audit line {line_number} is not an event") events.append(event) if not events: raise PreflightError("watchdog audit is empty") @@ -457,7 +663,7 @@ def _canonical_watchdog_audit( sleeper=sleeper, ) try: - heartbeat_record = json.loads(heartbeat_path.read_text(encoding="ascii")) + heartbeat_record = strict_json_loads(heartbeat_path.read_text(encoding="ascii")) updated = datetime.fromisoformat(str(heartbeat_record["updated_at"]).replace("Z", "+00:00")) heartbeat_unix = int(updated.timestamp()) except (OSError, UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: @@ -719,7 +925,38 @@ def matching_workloads( return matches -def run_preflight( +def darwin_matching_workloads( + patterns: list[str], + *, + command_text: Callable[..., str] = _command_text, + current_pid: int | None = None) -> list[dict[str, object]]: + pid_to_parent = {} + pid_to_command = {} + for line in command_text("ps", "-axo", "pid=,ppid=,command=").splitlines(): + fields = line.strip().split(maxsplit=2) + if len(fields) != 3: + continue + try: + pid = int(fields[0]) + parent = int(fields[1]) + except ValueError: + continue + pid_to_parent[pid] = parent + pid_to_command[pid] = fields[2] + excluded = set() + pid = os.getpid() if current_pid is None else current_pid + while pid > 0 and pid not in excluded: + excluded.add(pid) + pid = pid_to_parent.get(pid, 0) + lowered = [pattern.lower() for pattern in patterns if pattern] + return [ + {"pid": pid, "command": command} + for pid, command in pid_to_command.items() + if pid not in excluded and any(pattern in command.lower() for pattern in lowered) + ] + + +def run_strix_preflight( *, model: Path, prompt: Path, @@ -734,7 +971,13 @@ def run_preflight( tmpdir_value = os.environ.get("TMPDIR") if not tmpdir_value: raise PreflightError("TMPDIR is required for NVMe-only correctness runs") - tmp_storage = storage_attestation(Path(tmpdir_value), "temporary directory") + tmpdir_input = Path(tmpdir_value).expanduser().absolute() + if tmpdir_input.is_symlink(): + raise PreflightError("TMPDIR must not be a symlink") + tmpdir = resolved(tmpdir_input) + if not tmpdir.is_dir() or not os.access(tmpdir, os.W_OK | os.X_OK): + raise PreflightError("TMPDIR must be an existing writable directory") + tmp_storage = storage_attestation(tmpdir, "temporary directory") model = _attested_resolved_path(model_storage, "model") prompt = _attested_resolved_path(prompt_storage, "prompt") output = _attested_resolved_path(output_storage, "trace output") @@ -753,6 +996,7 @@ def run_preflight( raise PreflightError("active model workload detected: " + json.dumps(workloads, ensure_ascii=True)) return { "created_unix": int(time.time()), + "runtime_kind": "strix-rocm", "model": str(model), "prompt": str(prompt), "output": str(output), @@ -771,13 +1015,125 @@ def run_preflight( } +def run_oracle_preflight( + *, + model: Path, + prompt: Path, + output: Path, + repo: Path, + checkout: Path, + busy_patterns: list[str], + accelerator: dict[str, object], + runner: dict[str, object], + disk_info: Callable[[Path], dict[str, object]] = _diskutil_info, + command_text: Callable[..., str] = _command_text, + system: str | None = None, + machine: str | None = None) -> dict[str, object]: + model_storage = darwin_storage_attestation(model, "model", disk_info=disk_info) + prompt_storage = darwin_storage_attestation(prompt, "prompt", disk_info=disk_info) + output_storage = darwin_storage_attestation(output, "trace output", disk_info=disk_info) + repo_storage = darwin_storage_attestation(repo, "repository", disk_info=disk_info) + checkout_storage = darwin_storage_attestation(checkout, "ds4 checkout", disk_info=disk_info) + runner_executable = Path(str(runner.get("runner_executable", ""))) + runner_script = Path(str(runner.get("runner_script", ""))) + exporter = Path(str(runner.get("exporter_path", ""))) + runner_executable_storage = darwin_storage_attestation( + runner_executable, "runner executable", disk_info=disk_info) + runner_script_storage = darwin_storage_attestation(runner_script, "runner script", disk_info=disk_info) + exporter_storage = darwin_storage_attestation(exporter, "trace exporter", disk_info=disk_info) + tmpdir_value = os.environ.get("TMPDIR") + if not tmpdir_value: + raise PreflightError("TMPDIR is required for ds4 oracle correctness runs") + tmpdir_input = Path(tmpdir_value).expanduser().absolute() + if tmpdir_input.is_symlink(): + raise PreflightError("TMPDIR must not be a symlink") + tmpdir = resolved(tmpdir_input) + if not tmpdir.is_dir() or not os.access(tmpdir, os.W_OK | os.X_OK): + raise PreflightError("TMPDIR must be an existing writable directory") + tmp_storage = darwin_storage_attestation(tmpdir, "temporary directory", disk_info=disk_info) + model = _attested_resolved_path(model_storage, "model") + prompt = _attested_resolved_path(prompt_storage, "prompt") + output = _attested_resolved_path(output_storage, "trace output") + repo = _attested_resolved_path(repo_storage, "repository") + checkout = _attested_resolved_path(checkout_storage, "ds4 checkout") + runner_executable = _attested_resolved_path(runner_executable_storage, "runner executable") + runner_script = _attested_resolved_path(runner_script_storage, "runner script") + exporter = _attested_resolved_path(exporter_storage, "trace exporter") + if not model.is_file(): + raise PreflightError(f"model is not a file: {model}") + if not prompt.is_file(): + raise PreflightError(f"prompt is not a file: {prompt}") + if not runner_executable.is_file() or not runner_script.is_file() or not exporter.is_file(): + raise PreflightError("ds4 runner or exporter path is not a file") + try: + runner_script.relative_to(repo) + except ValueError as error: + raise PreflightError("ds4 runner script is outside the attested repository") from error + for key, path in ( + ("runner_executable_sha256", runner_executable), + ("runner_script_sha256", runner_script), + ("exporter_sha256", exporter)): + try: + digest = sha256_bytes(path.read_bytes()) + except OSError as error: + raise PreflightError(f"cannot hash ds4 {key} path: {error}") from error + if runner.get(key) != digest: + raise PreflightError(f"ds4 {key} differs from the executed file") + if runner.get("checkout_path") != str(checkout): + raise PreflightError("ds4 runner checkout path differs from the attested checkout") + host, memory = darwin_host_and_memory_audit( + command_text=command_text, + system=system, + machine=machine, + ) + swap = darwin_swap_audit(command_text=command_text) + workloads = darwin_matching_workloads(busy_patterns, command_text=command_text) + if workloads: + raise PreflightError("active model workload detected: " + json.dumps(workloads, ensure_ascii=True)) + return { + "created_unix": int(time.time()), + "runtime_kind": "apple-metal", + "model": str(model), + "prompt": str(prompt), + "output": str(output), + "memory": memory, + "swap": swap, + "runner": runner, + "host": host, + "accelerator": accelerator, + "active_workloads": [], + "environment": {}, + "storage": { + "model": model_storage, + "prompt": prompt_storage, + "output": output_storage, + "repository": repo_storage, + "runtime_checkout": checkout_storage, + "temporary_directory": tmp_storage, + "runner_executable": runner_executable_storage, + "runner_script": runner_script_storage, + "exporter": exporter_storage, + }, + } + + def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: root = resolved(root) if root.exists() and any(root.iterdir()): raise PreflightError(f"audit directory is not empty: {root}") root.mkdir(parents=True, exist_ok=True) result = {} - for key in ("memory", "swap", "watchdog"): + runtime_kind = audit.get("runtime_kind") + kinds = ( + ("memory", "swap", "watchdog") + if runtime_kind == "strix-rocm" + else ("memory", "swap", "runner") + if runtime_kind == "apple-metal" + else () + ) + if not kinds: + raise PreflightError("audit runtime kind is invalid") + for key in kinds: path = root / f"{key}.json" data = audit[key] if key == "watchdog": @@ -791,7 +1147,7 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: events = [] for line_number, line in enumerate(audit_text.splitlines(), start=1): try: - event = json.loads(line) + event = strict_json_loads(line) except json.JSONDecodeError as error: raise PreflightError( f"watchdog audit line {line_number} is invalid while snapshotting: {error}") from error @@ -815,8 +1171,9 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: } if key == "memory": value["storage"] = audit["storage"] - if "accelerator" in audit: - value["accelerator"] = audit["accelerator"] + value["accelerator"] = audit["accelerator"] + if "host" in audit: + value["host"] = audit["host"] path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") result[key] = str(path) summary = root / "preflight.json" @@ -829,13 +1186,16 @@ def seal_audits(audits: dict[str, str]) -> dict[str, str]: digests = {} paths = [] try: - for kind in ("memory", "swap", "watchdog"): + kinds = tuple(kind for kind in ("memory", "swap", "watchdog", "runner") if kind in audits) + if set(kinds) != set(audits) - {"preflight"}: + raise PreflightError("audit set has unsupported kinds") + for kind in kinds: path = resolved(Path(audits[kind])) paths.append(path) data = path.read_bytes() digests[kind] = sha256_bytes(data) if kind == "watchdog": - record = json.loads(data.decode("ascii")) + record = strict_json_loads(data.decode("ascii")) jsonl_path = resolved(Path(record["data"]["audit_path"])) paths.append(jsonl_path) digests["watchdog_jsonl"] = sha256_bytes(jsonl_path.read_bytes()) @@ -857,11 +1217,14 @@ def embed_audits(trace_root: Path, phase: str, audits: dict[str, str]) -> dict[s embedded_root = safe_trace_path(trace_root, Path("audits") / phase) embedded_root.mkdir(parents=True, exist_ok=True) result = {} - for kind in ("memory", "swap", "watchdog"): + kinds = tuple(kind for kind in ("memory", "swap", "watchdog", "runner") if kind in audits) + if set(kinds) != set(audits) - {"preflight"}: + raise PreflightError("audit set has unsupported kinds") + for kind in kinds: source = resolved(Path(audits[kind])) data = source.read_bytes() try: - record = json.loads(data.decode("ascii")) + record = strict_json_loads(data.decode("ascii")) except (UnicodeError, json.JSONDecodeError) as error: raise PreflightError(f"cannot embed {kind} audit: {error}") from error if kind == "watchdog": @@ -907,7 +1270,7 @@ def bind_embedded_audits(trace_root: Path, audit_sets: dict[str, dict[str, str]] trace_root = safe_trace_path(trace_root, ".") manifest_path = safe_trace_path(trace_root, "manifest.json") try: - manifest = json.loads(manifest_path.read_text(encoding="ascii")) + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise PreflightError(f"cannot bind trace audits: {error}") from error if set(audit_sets) != {"pre", "post"}: @@ -929,11 +1292,12 @@ def validate_prompt_provenance( corpus_sha256: str, model_sha256: str, target_tokens: int, + path_resolver: Callable[[Path, str], Path] | None = None, ) -> dict[str, object]: - path = require_nvme_path(path, "prompt provenance") + path = (require_nvme_path if path_resolver is None else path_resolver)(path, "prompt provenance") try: data = path.read_bytes() - record = json.loads(data.decode("ascii")) + record = strict_json_loads(data.decode("ascii")) prompt_path = resolved(prompt) prompt_bytes = prompt_path.read_bytes() prompt_size = prompt_path.stat().st_size @@ -965,7 +1329,7 @@ def bind_prompt_provenance(trace_root: Path, provenance: dict[str, object]) -> N trace_root = safe_trace_path(trace_root, ".") manifest_path = safe_trace_path(trace_root, "manifest.json") try: - manifest = json.loads(manifest_path.read_text(encoding="ascii")) + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise PreflightError(f"cannot bind prompt provenance: {error}") from error prompt = manifest.get("prompt") diff --git a/tools/deepseek-v41-trace/prompt-builder.cpp b/tools/deepseek-v41-trace/prompt-builder.cpp index cfd66b98f0e4..47b0c700785e 100644 --- a/tools/deepseek-v41-trace/prompt-builder.cpp +++ b/tools/deepseek-v41-trace/prompt-builder.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -52,6 +53,13 @@ int main(int argc, char ** argv) { model_path = dsv41::require_nvme_path(model_path, "model").resolved_path; corpus_path = dsv41::require_nvme_path(corpus_path, "corpus").resolved_path; output_path = dsv41::require_nvme_path(output_path, "prompt output").resolved_path; + const char * tmpdir_value = std::getenv("TMPDIR"); + if (tmpdir_value == nullptr || *tmpdir_value == '\0') { + throw std::runtime_error("TMPDIR is required"); + } + dsv41::require_usable_directory(tmpdir_value, "TMPDIR"); + const dsv41::storage_attestation temporary_storage = + dsv41::require_nvme_path(tmpdir_value, "temporary directory"); if (fs::exists(output_path)) { throw std::runtime_error("prompt output already exists: " + output_path.string()); } @@ -115,6 +123,7 @@ int main(int argc, char ** argv) { {"actual_tokens", verified.size()}, {"byte_count", prompt.size()}, {"add_bos", add_bos}, + {"temporary_directory", temporary_storage.resolved_path.string()}, }).dump().c_str()); llama_model_free(model); return 0; diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 7db2c0f71bf9..ee242d736151 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -12,14 +12,25 @@ PreflightError, bind_embedded_audits, bind_prompt_provenance, + darwin_storage_attestation, resolved, - run_preflight, + run_oracle_preflight, seal_audits, validate_prompt_provenance, verify_sealed_audits, write_audits, ) -from trace_format import ADMITTED_UBATCH, CORPUS_SHA256, MODEL_SHA256, TraceBundle, TraceError, sha256_file +from trace_format import ( + ADMITTED_UBATCH, + CORPUS_SHA256, + MODEL_SHA256, + TraceBundle, + TraceError, + canonical_json, + sha256_bytes, + sha256_file, + strict_json_loads, +) DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" APPROVED_EXPORTERS: dict[str, str] = {} @@ -51,17 +62,171 @@ def verify_exporter_approval(exporter_sha256: str) -> None: "publish and review the exporter before cross-runtime execution") -def preflight(args: argparse.Namespace) -> dict[str, object]: +def validate_accelerator_attestation( + record: object, + *, + expected_device: str = "Metal0") -> dict[str, object]: + if not isinstance(record, dict): + raise PreflightError("accelerator attestation is not an object") + required_keys = { + "format", + "version", + "runtime_kind", + "platform", + "backend", + "backend_device", + "backend_description", + "architecture", + "metal_registry_id", + "recommended_max_working_set_bytes", + "unified_memory", + "source", + } + if set(record) != required_keys: + raise PreflightError("accelerator attestation fields are invalid") + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "backend": "Metal", + "backend_device": expected_device, + "unified_memory": True, + "source": "metal-device-query", + } + for key, value in expected.items(): + if record.get(key) != value: + raise PreflightError(f"accelerator attestation {key} mismatch") + if type(record.get("unified_memory")) is not bool: + raise PreflightError("accelerator attestation unified-memory identity is invalid") + for key in ("backend_description", "architecture"): + if not isinstance(record.get(key), str) or not record[key]: + raise PreflightError(f"accelerator attestation {key} is missing") + if type(record.get("metal_registry_id")) is not int or record["metal_registry_id"] <= 0: + raise PreflightError("accelerator attestation Metal registry identity is invalid") + if type(record.get("recommended_max_working_set_bytes")) is not int or ( + record["recommended_max_working_set_bytes"] <= 0): + raise PreflightError("accelerator attestation working-set identity is invalid") + return dict(record) + + +def query_accelerator_attestation(exporter: Path, device: str) -> dict[str, object]: + try: + result = subprocess.run( + [str(exporter), "--dsv41-attest-device", device], + check=False, + capture_output=True, + text=True, + ) + except OSError as error: + raise PreflightError(f"cannot query selected accelerator: {error}") from error + if result.returncode != 0: + detail = result.stderr.strip() or f"exit {result.returncode}" + raise PreflightError(f"selected accelerator query failed: {detail}") + try: + record = strict_json_loads(result.stdout) + except TraceError as error: + raise PreflightError(f"selected accelerator query returned invalid JSON: {error}") from error + return validate_accelerator_attestation(record, expected_device=device) + + +def runner_attestation( + *, + exporter: Path, + exporter_sha256: str, + checkout: Path, + command: list[str]) -> dict[str, object]: + runner_executable = resolved(Path(sys.executable)) + runner_script = resolved(Path(__file__)) + return { + "format": "dsv41-runner-ownership", + "version": 1, + "runtime_kind": "apple-metal", + "source": "python-subprocess", + "runner_pid": os.getpid(), + "runner_parent_pid": os.getppid(), + "runner_uid": os.getuid(), + "runner_executable": str(runner_executable), + "runner_executable_sha256": sha256_file(runner_executable), + "runner_script": str(runner_script), + "runner_script_sha256": sha256_file(runner_script), + "exporter_path": str(exporter), + "exporter_sha256": exporter_sha256, + "checkout_path": str(checkout), + "checkout_revision": DS4_REVISION, + "command_sha256": sha256_bytes(canonical_json(command).encode("ascii")), + } + + +def bind_oracle_attestation( + output: Path, + audit: dict[str, object], + accelerator: dict[str, object], + command: list[str]) -> None: + manifest_path = output / "manifest.json" + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: + raise PreflightError(f"cannot bind ds4 runtime attestation: {error}") from error + if manifest.get("accelerator") != accelerator: + raise PreflightError("ds4 trace accelerator attestation differs from the preflight query") + storage = audit.get("storage") + if not isinstance(storage, dict): + raise PreflightError("ds4 storage attestation is missing") + paths = {} + for label, record in storage.items(): + if not isinstance(record, dict) or not isinstance(record.get("resolved_path"), str): + raise PreflightError(f"ds4 storage attestation is invalid for {label}") + paths[label] = record["resolved_path"] + if manifest.get("model", {}).get("path") != paths["model"]: + raise PreflightError("ds4 trace model path differs from the attested path") + if manifest.get("prompt", {}).get("path") != paths["prompt"]: + raise PreflightError("ds4 trace prompt path differs from the attested path") + if "paths" in manifest and manifest["paths"] != paths: + raise PreflightError("ds4 trace execution paths differ from preflight") + manifest["paths"] = paths + host = audit.get("host") + if not isinstance(host, dict): + raise PreflightError("ds4 host attestation is missing") + if "host" in manifest and manifest["host"] != host: + raise PreflightError("ds4 trace host identity differs from preflight") + manifest["host"] = host + manifest["environment"] = { + "system_info": f"macOS {host['os_version']} arm64 {host['hardware_model']}", + "command": shlex.join(command), + } + config = manifest.get("config") + if not isinstance(config, dict): + raise PreflightError("ds4 trace config is invalid") + for key, value in ( + ("device_backend", "Metal"), + ("device_registry_id", accelerator["metal_registry_id"])): + if key in config and config[key] != value: + raise PreflightError(f"ds4 trace {key} differs from preflight") + config[key] = value + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +def preflight( + args: argparse.Namespace, + *, + accelerator: dict[str, object], + runner: dict[str, object]) -> dict[str, object]: checkout = resolved(args.checkout) revision = verify_checkout(checkout) if revision != DS4_REVISION: raise PreflightError(f"ds4 revision mismatch: expected {DS4_REVISION}, found {revision}") - result = run_preflight( + result = run_oracle_preflight( model=args.model, prompt=args.prompt, output=args.output, repo=args.repo, + checkout=checkout, busy_patterns=args.busy_pattern, + accelerator=accelerator, + runner=runner, ) result.update({ "runtime": "ds4", @@ -71,6 +236,8 @@ def preflight(args: argparse.Namespace) -> dict[str, object]: "context": args.context, "decode_steps": args.decode_steps, "prefill_chunk": args.prefill_chunk, + "device_backend": "Metal", + "device_registry_id": accelerator["metal_registry_id"], }, }) return result @@ -92,6 +259,7 @@ def main() -> int: parser.add_argument("--context", type=int, default=32768) parser.add_argument("--decode-steps", type=int, default=8) parser.add_argument("--prefill-chunk", type=int, default=ADMITTED_UBATCH) + parser.add_argument("--device", default="Metal0") parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() @@ -102,11 +270,6 @@ def main() -> int: f"found {args.prefill_chunk}") if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") - if args.preflight_only: - audit = preflight(args) - print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) - return 0 - exporter = resolved(args.exporter) if not exporter.is_file() or not os.access(exporter, os.X_OK): raise PreflightError(f"trace exporter is not executable: {exporter}") @@ -115,6 +278,29 @@ def main() -> int: raise PreflightError( f"trace exporter SHA-256 mismatch: expected {args.exporter_sha256}, found {exporter_sha256}") verify_exporter_approval(exporter_sha256) + output = resolved(args.output) + command = [ + str(exporter), + "--model", str(resolved(args.model)), + "--prompt-file", str(resolved(args.prompt)), + "--output", str(output), + "--context", str(args.context), + "--decode-steps", str(args.decode_steps), + "--prefill-chunk", str(args.prefill_chunk), + "--device", args.device, + ] + accelerator = query_accelerator_attestation(exporter, args.device) + runner = runner_attestation( + exporter=exporter, + exporter_sha256=exporter_sha256, + checkout=resolved(args.checkout), + command=command, + ) + if args.preflight_only: + audit = preflight(args, accelerator=accelerator, runner=runner) + print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) + return 0 + model_sha256 = sha256_file(resolved(args.model)) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") @@ -125,35 +311,30 @@ def main() -> int: corpus_sha256=args.corpus_sha256, model_sha256=model_sha256, target_tokens=args.context - args.decode_steps, + path_resolver=lambda path, label: Path( + str(darwin_storage_attestation(path, label)["resolved_path"])), ) - output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") - preflight_audit = preflight(args) + preflight_audit = preflight(args, accelerator=accelerator, runner=runner) preflight_audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) pre_audit_digests = seal_audits(pre_audits) - command = [ - str(exporter), - "--model", str(resolved(args.model)), - "--prompt-file", str(resolved(args.prompt)), - "--output", str(output), - "--context", str(args.context), - "--decode-steps", str(args.decode_steps), - "--prefill-chunk", str(args.prefill_chunk), - "--memory-audit", pre_audits["memory"], - "--swap-audit", pre_audits["swap"], - "--watchdog-audit", pre_audits["watchdog"], - ] print("exec:", shlex.join(command), file=sys.stderr) result = subprocess.run(command, cwd=resolved(args.checkout), check=False) if result.returncode != 0: return result.returncode verify_sealed_audits(pre_audits, pre_audit_digests) - postflight_audit = preflight(args) + post_accelerator = query_accelerator_attestation(exporter, args.device) + if post_accelerator != accelerator: + raise PreflightError("selected accelerator identity changed during trace execution") + postflight_audit = preflight(args, accelerator=post_accelerator, runner=runner) + if postflight_audit.get("host") != preflight_audit.get("host"): + raise PreflightError("ds4 host identity changed during trace execution") post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) + bind_oracle_attestation(output, preflight_audit, accelerator, command) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "ds4": raise PreflightError("ds4 exporter wrote a non-ds4 trace") @@ -164,6 +345,8 @@ def main() -> int: raise PreflightError("ds4 trace build SHA-256 does not match the executed exporter") if bundle.manifest.get("model", {}).get("sha256") != MODEL_SHA256: raise PreflightError("ds4 trace model SHA-256 does not match the published GGUF") + if bundle.manifest.get("accelerator") != accelerator: + raise PreflightError("ds4 trace accelerator attestation differs from the measured Metal device") return 0 except (PreflightError, TraceError) as error: print(f"error: {error}", file=sys.stderr) diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index fd4aa5d2610d..72029a5aa50f 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -15,7 +15,7 @@ bind_embedded_audits, bind_prompt_provenance, resolved, - run_preflight, + run_strix_preflight, safe_trace_path, seal_audits, validate_prompt_provenance, @@ -34,6 +34,7 @@ TraceBundle, TraceError, sha256_file, + strict_json_loads, ) @@ -98,8 +99,8 @@ def bind_candidate_attestation( accelerator: dict[str, object]) -> None: manifest_path = safe_trace_path(output, "manifest.json") try: - manifest = json.loads(manifest_path.read_text(encoding="ascii")) - except (OSError, UnicodeError, json.JSONDecodeError) as error: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: raise PreflightError(f"cannot bind candidate attestation: {error}") from error if manifest.get("accelerator") != accelerator: raise PreflightError("llama trace accelerator attestation differs from the preflight query") @@ -115,9 +116,29 @@ def validate_accelerator_attestation( expected_device: str = "ROCm0") -> dict[str, object]: if not isinstance(record, dict): raise PreflightError("accelerator attestation is not an object") + required_keys = { + "format", + "version", + "runtime_kind", + "platform", + "backend", + "backend_device", + "backend_description", + "pci_device_id", + "kfd_node", + "gpu_id", + "gfx_target_version", + "architecture", + "source", + } + if set(record) != required_keys: + raise PreflightError("accelerator attestation fields are invalid") expected = { "format": "dsv41-accelerator-attestation", - "version": 1, + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "backend": "ROCm", "backend_device": expected_device, "architecture": "gfx1151", "gfx_target_version": 110501, @@ -134,7 +155,7 @@ def validate_accelerator_attestation( raise PreflightError("accelerator attestation PCI identity is invalid") if not isinstance(record.get("kfd_node"), str) or not record["kfd_node"].isdigit(): raise PreflightError("accelerator attestation KFD node is invalid") - if not isinstance(record.get("gpu_id"), int) or record["gpu_id"] <= 0: + if type(record.get("gpu_id")) is not int or record["gpu_id"] <= 0: raise PreflightError("accelerator attestation GPU identity is invalid") return dict(record) @@ -153,8 +174,8 @@ def query_accelerator_attestation(exporter: Path, device: str) -> dict[str, obje detail = result.stderr.strip() or f"exit {result.returncode}" raise PreflightError(f"selected accelerator query failed: {detail}") try: - record = json.loads(result.stdout) - except json.JSONDecodeError as error: + record = strict_json_loads(result.stdout) + except TraceError as error: raise PreflightError(f"selected accelerator query returned invalid JSON: {error}") from error return validate_accelerator_attestation(record, expected_device=device) @@ -235,7 +256,7 @@ def main() -> int: raise PreflightError(f"trace exporter is not executable: {exporter}") accelerator = query_accelerator_attestation(exporter, args.device) if args.preflight_only: - audit = run_preflight( + audit = run_strix_preflight( model=args.model, prompt=args.prompt, output=args.output, @@ -262,7 +283,7 @@ def main() -> int: output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") - preflight_audit = run_preflight( + preflight_audit = run_strix_preflight( model=args.model, prompt=args.prompt, output=args.output, @@ -295,7 +316,7 @@ def main() -> int: if result.returncode != 0: return result.returncode verify_sealed_audits(pre_audits, pre_audit_digests) - postflight_audit = run_preflight( + postflight_audit = run_strix_preflight( model=args.model, prompt=args.prompt, output=args.output, diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index 83ec25d9cdd3..fcae024f95c7 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -8,7 +8,7 @@ import sys from pathlib import Path -from preflight import PreflightError, require_nvme_path, resolved, run_preflight +from preflight import PreflightError, require_nvme_path, resolved, run_strix_preflight from trace_format import ( ADMITTED_BATCH, ADMITTED_UBATCH, @@ -16,9 +16,9 @@ MODEL_SHA256, REQUIRED_EXPERT_CACHE_MIB, REQUIRED_EXPERT_SLOTS, - TraceBundle, - report, + TraceError, sha256_file, + strict_json_loads, ) CORPORA = ( @@ -58,11 +58,15 @@ def prepare_prompt( if result.returncode != 0: raise RuntimeError(f"prompt builder failed: {result.stderr.strip()}") try: - record = json.loads(result.stdout) - except json.JSONDecodeError as error: + record = strict_json_loads(result.stdout) + except TraceError as error: raise RuntimeError(f"prompt builder returned invalid JSON: {error}") from error if record.get("actual_tokens") != target_tokens: raise RuntimeError("prompt builder did not produce the requested token count") + temporary_directory = record.pop("temporary_directory", None) + expected_temporary_directory = os.environ.get("TMPDIR") + if not expected_temporary_directory or temporary_directory != str(resolved(Path(expected_temporary_directory))): + raise RuntimeError("prompt builder did not attest the selected temporary directory") record.update({ "format": "dsv41-prompt-provenance", "version": 1, @@ -87,7 +91,7 @@ def prepare_prompt( def main() -> int: - parser = argparse.ArgumentParser(description="Run the DeepSeek V4.1 cross-runtime corpus matrix") + parser = argparse.ArgumentParser(description="Capture the DeepSeek V4.1 llama.cpp corpus matrix") parser.add_argument("--repo", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) @@ -97,10 +101,6 @@ def main() -> int: parser.add_argument("--candidate-revision", required=True) parser.add_argument("--base-revision", required=True) parser.add_argument("--candidate-diff-sha256", required=True) - parser.add_argument("--ds4-runner", type=Path) - parser.add_argument("--ds4-exporter", type=Path) - parser.add_argument("--ds4-exporter-sha256") - parser.add_argument("--ds4-checkout", type=Path, default=Path("/home/papa/src/ds4-v41")) parser.add_argument("--llama-only", action="store_true") parser.add_argument("--contexts", type=int, nargs="+", default=[32768]) parser.add_argument("--ubatches", type=int, nargs="+", default=[ADMITTED_UBATCH]) @@ -126,11 +126,10 @@ def main() -> int: if args.expert_cache_mib != REQUIRED_EXPERT_CACHE_MIB: raise PreflightError( f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_CACHE_MIB} MiB expert cache") - if not args.llama_only and ( - args.ds4_runner is None or args.ds4_exporter is None or args.ds4_exporter_sha256 is None): + if not args.llama_only: raise PreflightError( - "--ds4-runner, --ds4-exporter, and --ds4-exporter-sha256 are required " - "unless --llama-only is selected") + "cross-runtime capture must run on separate Strix and Apple hosts; " + "use --llama-only here and compare completed bundles with trace_format.py") repo = resolved(args.repo) output = require_nvme_path(args.output, "matrix output") model = require_nvme_path(args.model, "model") @@ -143,7 +142,7 @@ def main() -> int: repo / "tests" / "corpus" / CORPORA[0], "repository corpus", ) - run_preflight( + run_strix_preflight( model=model, prompt=initial_corpus, output=output, @@ -191,7 +190,7 @@ def main() -> int: for corpus in corpus_records: stem = Path(corpus["name"]).stem prompt = prompts / f"{stem}-c{context}.txt" - run_preflight( + run_strix_preflight( model=model, prompt=Path(corpus["path"]), output=prompt, @@ -215,7 +214,6 @@ def main() -> int: stem = Path(corpus["name"]).stem case = f"{stem}-c{context}-ub{ubatch}" llama_output = output / "llama" / case - ds4_output = output / "ds4" / case prompt = prepared_prompts[corpus["name"]]["path"] provenance = prepared_prompts[corpus["name"]]["provenance_path"] common = [ @@ -229,21 +227,6 @@ def main() -> int: ] for pattern in args.busy_pattern: common.extend(["--busy-pattern", pattern]) - if not args.llama_only: - assert args.ds4_runner is not None - assert args.ds4_exporter is not None - assert args.ds4_exporter_sha256 is not None - run([ - sys.executable, - str(resolved(args.ds4_runner)), - "--repo", str(repo), - "--checkout", str(resolved(args.ds4_checkout)), - "--exporter", str(resolved(args.ds4_exporter)), - "--exporter-sha256", args.ds4_exporter_sha256, - "--output", str(ds4_output), - "--prefill-chunk", str(ubatch), - *common, - ]) run([ sys.executable, str(resolved(args.llama_runner)), @@ -260,30 +243,17 @@ def main() -> int: "--expert-cache-mib", str(args.expert_cache_mib), *common, ]) - if args.llama_only: - results.append({ - "case": case, - "status": "BRINGUP TRACE CAPTURED", - "cross_runtime_status": "INCOMPLETE", - "trace": str(llama_output), - }) - else: - comparison = report(TraceBundle(ds4_output), TraceBundle(llama_output)) - result_path = output / "reports" / f"{case}.json" - result_path.parent.mkdir(parents=True, exist_ok=True) - result_path.write_text( - json.dumps(comparison, sort_keys=True, separators=(",", ":")) + "\n", - encoding="ascii", - ) - results.append({"case": case, **comparison}) - if comparison["status"] != "TARGET PASS": - raise RuntimeError( - f"correctness mismatch in {case}: {comparison['first_divergence']}") + results.append({ + "case": case, + "status": "BRINGUP TRACE CAPTURED", + "cross_runtime_status": "INCOMPLETE", + "trace": str(llama_output), + }) summary = { - "status": "BRINGUP TRACE CAPTURED" if args.llama_only else "TARGET PASS", - "mode": "llama-only" if args.llama_only else "cross-runtime", - "cross_runtime_status": "INCOMPLETE" if args.llama_only else "TARGET PASS", + "status": "BRINGUP TRACE CAPTURED", + "mode": "llama-only", + "cross_runtime_status": "INCOMPLETE", "model": str(model), "model_sha256": model_sha256, "candidate_revision": args.candidate_revision, diff --git a/tools/deepseek-v41-trace/test-host-attestation.cpp b/tools/deepseek-v41-trace/test-host-attestation.cpp index dbe2ec349617..8459ce9d9281 100644 --- a/tools/deepseek-v41-trace/test-host-attestation.cpp +++ b/tools/deepseek-v41-trace/test-host-attestation.cpp @@ -42,12 +42,14 @@ int main() { const fs::path ram = root / "ram"; const fs::path network = root / "network"; const fs::path missing = root / "missing"; + const fs::path forbidden = root / "forbidden"; fs::create_directories(xfs); fs::create_directories(btrfs); fs::create_directories(rotating); fs::create_directories(ram); fs::create_directories(network); fs::create_directories(missing); + fs::create_directories(forbidden); write_file(xfs / "model.gguf", "model"); write_file(btrfs / "corpus.txt", "corpus"); write_file(rotating / "data", "data"); @@ -55,6 +57,14 @@ int main() { write_file(network / "data", "data"); write_file(missing / "data", "data"); fs::create_directory_symlink(ram, xfs / "escape"); + fs::create_directory_symlink(xfs, forbidden / "escape"); + fs::create_directory_symlink(btrfs, root / "tmp-link"); + dsv41::require_usable_directory(btrfs, "TMPDIR"); + require_failure( + [&]() { + dsv41::require_usable_directory(root / "tmp-link", "TMPDIR"); + }, + "must not be a symlink"); const fs::path sys = root / "sys"; const fs::path nvme1 = sys / "devices" / "pci" / "block" / "nvme1n1"; @@ -135,6 +145,13 @@ int main() { sys / "class" / "block"); }, "/mnt/bigspace"); + require_failure( + [&]() { + dsv41::require_nvme_path( + forbidden / "escape" / "model.gguf", "forbidden symlink", mountinfo, + sys / "dev" / "block", sys / "class" / "block", forbidden); + }, + "must not use"); const fs::path kfd = root / "kfd"; write_file( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index a3eb67307cd6..4906f7bbda8e 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -3,6 +3,7 @@ import argparse import hashlib import json +import math import os import re import struct @@ -138,10 +139,363 @@ def canonical_json(data: Any) -> str: return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=True) +def strict_json_loads(data: str) -> Any: + def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result = {} + for key, value in pairs: + if key in result: + raise TraceError(f"duplicate JSON key: {key}") + result[key] = value + return result + + try: + return json.loads(data, object_pairs_hook=reject_duplicates) + except json.JSONDecodeError as error: + raise TraceError(f"invalid JSON: {error}") from error + + +def _require_exact_keys(record: dict[str, Any], keys: set[str], label: str) -> None: + missing = sorted(keys - set(record)) + extra = sorted(set(record) - keys) + if missing or extra: + detail = [] + if missing: + detail.append("missing " + ", ".join(missing)) + if extra: + detail.append("unexpected " + ", ".join(extra)) + raise TraceError(f"{label} fields are invalid: {'; '.join(detail)}") + + +def validate_accelerator_attestation(runtime: str, accelerator: Any) -> dict[str, Any]: + if not isinstance(accelerator, dict): + raise TraceError("manifest accelerator attestation is invalid") + common = { + "format", + "version", + "runtime_kind", + "platform", + "backend", + "backend_device", + "backend_description", + "architecture", + "source", + } + if runtime == "llama.cpp": + _require_exact_keys( + accelerator, + common | {"pci_device_id", "kfd_node", "gpu_id", "gfx_target_version"}, + "llama.cpp accelerator attestation", + ) + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "backend": "ROCm", + "backend_device": "ROCm0", + "architecture": "gfx1151", + "gfx_target_version": 110501, + "source": "linux-kfd-sysfs", + } + for key, value in expected.items(): + if accelerator.get(key) != value: + raise TraceError(f"llama.cpp accelerator {key} mismatch") + if re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", + accelerator.get("pci_device_id", "")) is None: + raise TraceError("llama.cpp accelerator PCI identity is invalid") + if not isinstance(accelerator.get("kfd_node"), str) or not accelerator["kfd_node"].isdigit(): + raise TraceError("llama.cpp accelerator KFD node is invalid") + if type(accelerator.get("gpu_id")) is not int or accelerator["gpu_id"] <= 0: + raise TraceError("llama.cpp accelerator GPU identity is invalid") + elif runtime == "ds4": + _require_exact_keys( + accelerator, + common | { + "metal_registry_id", + "recommended_max_working_set_bytes", + "unified_memory", + }, + "ds4 accelerator attestation", + ) + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "backend": "Metal", + "source": "metal-device-query", + "unified_memory": True, + } + for key, value in expected.items(): + if accelerator.get(key) != value: + raise TraceError(f"ds4 accelerator {key} mismatch") + if type(accelerator.get("unified_memory")) is not bool: + raise TraceError("ds4 accelerator unified-memory identity is invalid") + registry_id = accelerator.get("metal_registry_id") + if type(registry_id) is not int or registry_id <= 0: + raise TraceError("ds4 accelerator Metal registry identity is invalid") + working_set = accelerator.get("recommended_max_working_set_bytes") + if type(working_set) is not int or working_set <= 0: + raise TraceError("ds4 accelerator working-set identity is invalid") + else: + raise TraceError(f"unsupported runtime accelerator attestation: {runtime}") + for key in ("backend_device", "backend_description", "architecture"): + if not isinstance(accelerator.get(key), str) or not accelerator[key]: + raise TraceError(f"manifest accelerator {key} is invalid") + return accelerator + + +def validate_storage_attestation(runtime: str, item: Any) -> dict[str, Any]: + if not isinstance(item, dict): + raise TraceError("storage attestation is invalid") + common = { + "format", + "version", + "runtime_kind", + "platform", + "storage_kind", + "resolved_path", + "existing_path", + "mount_point", + "filesystem_type", + "source", + } + if runtime == "llama.cpp": + _require_exact_keys( + item, + common | { + "mount_source", + "device_number", + "block_device_path", + "nvme_device", + "rotational", + }, + "llama.cpp storage attestation", + ) + expected = { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "storage_kind": "linux-nvme", + "source": "linux-mountinfo-sysfs", + "rotational": False, + } + for key, value in expected.items(): + if item.get(key) != value: + raise TraceError(f"llama.cpp storage {key} mismatch") + if type(item.get("rotational")) is not bool: + raise TraceError("llama.cpp storage rotational identity is invalid") + if not isinstance(item.get("nvme_device"), str) or re.fullmatch( + r"nvme[0-9]+(?:c[0-9]+)?n[0-9]+", item["nvme_device"]) is None: + raise TraceError("llama.cpp storage NVMe device identity is invalid") + if not isinstance(item.get("mount_source"), str) or not item["mount_source"].startswith("/dev/"): + raise TraceError("llama.cpp storage mount source is not a local block device") + if re.fullmatch(r"[0-9]+:[0-9]+", item.get("device_number", "")) is None: + raise TraceError("llama.cpp storage device number is invalid") + block_device_path = item.get("block_device_path") + if not isinstance(block_device_path, str) or item["nvme_device"] not in Path(block_device_path).parts: + raise TraceError("llama.cpp storage block device ancestry is invalid") + elif runtime == "ds4": + _require_exact_keys( + item, + common | { + "device_identifier", + "parent_whole_disk", + "bus_protocol", + "filesystem_device", + "internal", + "solid_state", + }, + "ds4 storage attestation", + ) + expected = { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "storage_kind": "darwin-local-solid-state", + "source": "diskutil-info-plist", + "internal": True, + "solid_state": True, + } + for key, value in expected.items(): + if item.get(key) != value: + raise TraceError(f"ds4 storage {key} mismatch") + if type(item.get("internal")) is not bool or type(item.get("solid_state")) is not bool: + raise TraceError("ds4 storage media identity is invalid") + for key in ("device_identifier", "parent_whole_disk", "bus_protocol"): + if not isinstance(item.get(key), str) or not item[key]: + raise TraceError(f"ds4 storage {key} is invalid") + if item["bus_protocol"].lower() in {"network", "virtual", "disk image"}: + raise TraceError("ds4 storage bus protocol is not local") + if type(item.get("filesystem_device")) is not int or item["filesystem_device"] < 0: + raise TraceError("ds4 storage filesystem device identity is invalid") + else: + raise TraceError(f"unsupported runtime storage attestation: {runtime}") + for path_key in ("resolved_path", "existing_path", "mount_point"): + value = item.get(path_key) + if not isinstance(value, str) or not value.startswith("/"): + raise TraceError(f"storage {path_key} is invalid") + if not isinstance(item.get("filesystem_type"), str) or not item["filesystem_type"]: + raise TraceError("storage filesystem type is invalid") + try: + Path(item["resolved_path"]).relative_to(Path(item["mount_point"])) + Path(item["existing_path"]).relative_to(Path(item["mount_point"])) + except ValueError as error: + raise TraceError("storage mount ancestry is invalid") from error + return item + + +def validate_host_attestation(host: Any) -> dict[str, Any]: + if not isinstance(host, dict): + raise TraceError("ds4 host attestation is invalid") + _require_exact_keys( + host, + { + "format", + "version", + "runtime_kind", + "platform", + "machine", + "hardware_model", + "os_version", + "memory_bytes", + "source", + }, + "ds4 host attestation", + ) + expected = { + "format": "dsv41-host-attestation", + "version": 1, + "runtime_kind": "apple-metal", + "platform": "macos", + "machine": "arm64", + "source": "darwin-sysctl", + } + for key, value in expected.items(): + if host.get(key) != value: + raise TraceError(f"ds4 host {key} mismatch") + for key in ("hardware_model", "os_version"): + if not isinstance(host.get(key), str) or not host[key]: + raise TraceError(f"ds4 host {key} is invalid") + if type(host.get("memory_bytes")) is not int or host["memory_bytes"] < 128 * 1024 * 1024 * 1024: + raise TraceError("ds4 host memory is below 128 GiB") + return host + + +WATCHDOG_STATE_KEYS = { + "total_bytes", + "available_bytes", + "used_bytes", + "swap_entries", + "peak_used_bytes", + "child_pid", + "child_status", + "child_returncode", + "process_group_id", + "process_group_status", + "threshold_reason", +} + + +def validate_watchdog_event(event: Any) -> dict[str, Any]: + if not isinstance(event, dict): + raise TraceError("watchdog JSONL event is not an object") + event_name = event.get("event") + required = {"timestamp", "event"} | WATCHDOG_STATE_KEYS + optional: set[str] = set() + if event_name == "preflight": + required |= {"soft_bytes", "emergency_bytes", "strict_ceiling_bytes"} + elif event_name == "child_started": + required.add("command") + elif event_name == "sample": + pass + elif event_name == "process_group_signal": + required.add("signal") + optional.add("grace_deadline_monotonic") + elif event_name == "final": + required |= {"classification", "exit_code"} + optional |= {"error", "secondary_errors"} + else: + raise TraceError(f"watchdog JSONL event name is invalid: {event_name}") + missing = sorted(required - set(event)) + unexpected = sorted(set(event) - required - optional) + if missing or unexpected: + details = [] + if missing: + details.append("missing " + ", ".join(missing)) + if unexpected: + details.append("unexpected " + ", ".join(unexpected)) + raise TraceError("watchdog JSONL event fields are invalid: " + "; ".join(details)) + if not isinstance(event["timestamp"], str) or re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z", + event["timestamp"]) is None: + raise TraceError("watchdog JSONL timestamp is invalid") + for key in ("total_bytes", "available_bytes", "used_bytes", "swap_entries", "peak_used_bytes"): + if event[key] is not None and (type(event[key]) is not int or event[key] < 0): + raise TraceError(f"watchdog JSONL {key} is invalid") + for key in ("child_pid", "process_group_id"): + if event[key] is not None and (type(event[key]) is not int or event[key] <= 0): + raise TraceError(f"watchdog JSONL {key} is invalid") + if event["child_returncode"] is not None and type(event["child_returncode"]) is not int: + raise TraceError("watchdog JSONL child_returncode is invalid") + if event["child_status"] not in {"not_started", "running", "signaled", "exited"}: + raise TraceError("watchdog JSONL child_status is invalid") + if event["process_group_status"] not in { + "not_created", "active", "leader_exited", "signal_error", + "termination_timeout", "missing", "sighup_sent", "sigint_sent", + "sigterm_sent", "sigkill_sent"}: + raise TraceError("watchdog JSONL process_group_status is invalid") + if not isinstance(event["threshold_reason"], str) or not event["threshold_reason"]: + raise TraceError("watchdog JSONL threshold_reason is invalid") + if event_name == "preflight": + for key in ("soft_bytes", "emergency_bytes", "strict_ceiling_bytes"): + if type(event[key]) is not int or event[key] <= 0: + raise TraceError(f"watchdog JSONL {key} is invalid") + elif event_name == "child_started": + if not isinstance(event["command"], list) or not event["command"] or ( + not all(isinstance(value, str) and value for value in event["command"])): + raise TraceError("watchdog JSONL command is invalid") + elif event_name == "process_group_signal": + if event["signal"] not in {"SIGHUP", "SIGINT", "SIGTERM", "SIGKILL"}: + raise TraceError("watchdog JSONL signal is invalid") + deadline = event.get("grace_deadline_monotonic") + if deadline is not None and ( + event["signal"] != "SIGTERM" or + not isinstance(deadline, (int, float)) or isinstance(deadline, bool) or + not math.isfinite(deadline)): + raise TraceError("watchdog JSONL grace deadline is invalid") + elif event_name == "final": + if event["classification"] not in { + "child_exit", "soft_limit", "grace_timeout", "swap_appeared", + "emergency_limit", "procfs_error", "signal_error", "termination_timeout", + "lease_error", "parent_signal", "internal_error", "launch_error", + "configuration_error", "startup_swap_active", "startup_emergency_limit", + "startup_soft_limit"}: + raise TraceError("watchdog JSONL classification is invalid") + if type(event["exit_code"]) is not int: + raise TraceError("watchdog JSONL exit code is invalid") + if "error" in event and (not isinstance(event["error"], str) or not event["error"]): + raise TraceError("watchdog JSONL error is invalid") + secondary_errors = event.get("secondary_errors") + if secondary_errors is not None: + if not isinstance(secondary_errors, list) or not secondary_errors: + raise TraceError("watchdog JSONL secondary errors are invalid") + for secondary_error in secondary_errors: + _require_exact_keys( + secondary_error, {"component", "detail"}, "watchdog JSONL secondary error") + if secondary_error["component"] not in {"audit", "lease", "stderr"} or ( + not isinstance(secondary_error["detail"], str) or not secondary_error["detail"]): + raise TraceError("watchdog JSONL secondary error is invalid") + return event + + def element_count(shape: Iterable[int]) -> int: count = 1 for dim in shape: - if not isinstance(dim, int) or dim <= 0: + if type(dim) is not int or dim <= 0: raise TraceError(f"shape dimension must be a nonzero positive integer: {dim!r}") count *= dim return count @@ -233,19 +587,25 @@ def validate_event(event: dict[str, Any]) -> None: missing = sorted(required - event.keys()) if missing: raise TraceError(f"event is missing fields: {', '.join(missing)}") + allowed = set(required) + if event.get("component") == "expert.ids": + allowed.add("semantic_id_space") + extra = sorted(set(event) - allowed) + if extra: + raise TraceError(f"event has unexpected fields: {', '.join(extra)}") if event["trace_version"] != TRACE_VERSION: raise TraceError(f"unsupported event version: {event['trace_version']!r}") if event["byte_order"] != "little": raise TraceError("trace blobs must use little-endian byte order") if event["phase"] not in ("input", "prefill", "decode"): raise TraceError("event phase is invalid") - if not isinstance(event["step"], int) or event["step"] < 0: + if type(event["step"]) is not int or event["step"] < 0: raise TraceError("event step is invalid") - if not isinstance(event["token_start"], int) or event["token_start"] < 0: + if type(event["token_start"]) is not int or event["token_start"] < 0: raise TraceError("event token_start is invalid") - if not isinstance(event["token_count"], int) or event["token_count"] <= 0: + if type(event["token_count"]) is not int or event["token_count"] <= 0: raise TraceError("event token_count is invalid") - if event["layer"] is not None and (not isinstance(event["layer"], int) or event["layer"] < 0): + if event["layer"] is not None and (type(event["layer"]) is not int or event["layer"] < 0): raise TraceError("event layer is invalid") if not isinstance(event["shape"], list) or not event["shape"] or any(dim <= 0 for dim in event["shape"]): raise TraceError("event shape dimensions must be nonzero") @@ -350,8 +710,8 @@ def __init__(self, root: Path, verify_blobs: bool = True): raise TraceError("trace root must not be a symlink") self.root = root.resolve() try: - self.manifest = json.loads(self._path(MANIFEST_NAME).read_text(encoding="ascii")) - except (OSError, UnicodeError, json.JSONDecodeError) as error: + self.manifest = strict_json_loads(self._path(MANIFEST_NAME).read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: raise TraceError(f"cannot read manifest: {error}") from error if self.manifest.get("trace_format") != TRACE_FORMAT: raise TraceError("manifest trace_format mismatch") @@ -372,8 +732,8 @@ def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: if not raw.endswith(b"\n"): raise TraceError(f"events.jsonl is truncated at line {line_number}") try: - event = json.loads(raw.decode("ascii")) - except (UnicodeError, json.JSONDecodeError) as error: + event = strict_json_loads(raw.decode("ascii")) + except (UnicodeError, TraceError) as error: raise TraceError(f"invalid event at line {line_number}: {error}") from error validate_event(event) if verify_blobs: @@ -388,69 +748,127 @@ def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: return result def _validate_manifest(self) -> None: - for key in ( - "runtime", "revision", "build", "model", "prompt", "accelerator", - "config", "comparison", "environment", "audits"): - if key not in self.manifest: - raise TraceError(f"manifest is missing {key}") - if not isinstance(self.manifest["runtime"], str) or not self.manifest["runtime"]: + if not isinstance(self.manifest.get("runtime"), str) or not self.manifest["runtime"]: raise TraceError("manifest runtime is invalid") if self.manifest["runtime"] not in ("ds4", "llama.cpp"): raise TraceError("manifest runtime must be ds4 or llama.cpp") + top_level = { + "trace_format", + "trace_version", + "event_count", + "runtime", + "revision", + "build", + "model", + "prompt", + "accelerator", + "config", + "comparison", + "environment", + "paths", + "audits", + "expected", + } + top_level.add("candidate" if self.manifest["runtime"] == "llama.cpp" else "host") + _require_exact_keys(self.manifest, top_level, f"{self.manifest['runtime']} manifest") if not isinstance(self.manifest["revision"], str) or not self.manifest["revision"]: raise TraceError("manifest revision is invalid") if self.manifest["runtime"] == "ds4" and self.manifest["revision"] != DS4_REVISION: raise TraceError(f"ds4 revision must be {DS4_REVISION}") if not isinstance(self.manifest["build"], dict): raise TraceError("manifest build is invalid") + build_keys = ( + {"number", "info", "compiler", "target", "path", "sha256"} + if self.manifest["runtime"] == "llama.cpp" + else {"compiler", "target", "path", "sha256"} + ) + _require_exact_keys(self.manifest["build"], build_keys, "manifest build") build_sha256 = self.manifest["build"].get("sha256", "") if not isinstance(build_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", build_sha256) is None: raise TraceError("manifest build SHA-256 is invalid") + for key in build_keys - {"sha256", "number"}: + value = self.manifest["build"].get(key) + if not isinstance(value, str) or not value: + raise TraceError(f"manifest build {key} is invalid") + if "number" in build_keys and type(self.manifest["build"]["number"]) is not int: + raise TraceError("manifest build number is invalid") + if not self.manifest["build"]["path"].startswith("/"): + raise TraceError("manifest build path is not absolute") for section in ("model", "prompt"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") + _require_exact_keys( + self.manifest["model"], + {"path", "sha256", "byte_count", "architecture"}, + "manifest model", + ) + _require_exact_keys( + self.manifest["prompt"], + { + "path", + "sha256", + "byte_count", + "corpus_name", + "corpus_sha256", + "target_tokens", + "provenance", + }, + "manifest prompt", + ) + for section in ("model", "prompt"): digest = self.manifest[section].get("sha256") if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError(f"manifest {section} SHA-256 is invalid") - if not isinstance(self.manifest[section].get("byte_count"), int) or self.manifest[section]["byte_count"] <= 0: + if type(self.manifest[section].get("byte_count")) is not int or self.manifest[section]["byte_count"] <= 0: raise TraceError(f"manifest {section} byte_count is invalid") for section in ("config", "comparison", "environment", "audits"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") + _require_exact_keys( + self.manifest["environment"], + {"system_info", "command"}, + "manifest environment", + ) + if not all(isinstance(value, str) and value for value in self.manifest["environment"].values()): + raise TraceError("manifest environment values are invalid") + system_info = self.manifest["environment"]["system_info"].lower() + if self.manifest["runtime"] == "llama.cpp" and "linux" not in system_info: + raise TraceError("llama.cpp environment is not Linux") + if self.manifest["runtime"] == "ds4" and not any( + name in system_info for name in ("darwin", "macos")): + raise TraceError("ds4 environment is not macOS") context = self.manifest["config"].get("context") decode_steps = self.manifest["config"].get("decode_steps") - if not isinstance(context, int) or not isinstance(decode_steps, int) or ( + if type(context) is not int or type(decode_steps) is not int or ( decode_steps <= 0 or context <= decode_steps): raise TraceError("manifest context or decode_steps is invalid") if self.manifest["model"]["sha256"] != MODEL_SHA256: raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") if self.manifest["model"].get("architecture") != "deepseek41": raise TraceError("model architecture must be deepseek41") - accelerator = self.manifest["accelerator"] - if not isinstance(accelerator, dict): - raise TraceError("manifest accelerator attestation is invalid") - accelerator_expected = { - "format": "dsv41-accelerator-attestation", - "version": 1, - "architecture": "gfx1151", - "gfx_target_version": 110501, - "source": "linux-kfd-sysfs", + accelerator = validate_accelerator_attestation(self.manifest["runtime"], self.manifest["accelerator"]) + if self.manifest["runtime"] == "ds4": + if "host" not in self.manifest: + raise TraceError("ds4 host attestation is missing") + validate_host_attestation(self.manifest["host"]) + else: + if "host" in self.manifest: + raise TraceError("llama.cpp manifest must not contain Apple host attestation") + required_paths = { + "model", "prompt", "output", "repository", "temporary_directory", } - for key, value in accelerator_expected.items(): - if accelerator.get(key) != value: - raise TraceError(f"manifest accelerator {key} mismatch") - if not isinstance(accelerator.get("backend_description"), str) or not accelerator["backend_description"]: - raise TraceError("manifest accelerator backend description is invalid") - if not isinstance(accelerator.get("backend_device"), str) or not accelerator["backend_device"]: - raise TraceError("manifest accelerator backend device is invalid") - if re.fullmatch( - r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", - accelerator.get("pci_device_id", "")) is None: - raise TraceError("manifest accelerator PCI identity is invalid") - if not isinstance(accelerator.get("kfd_node"), str) or not accelerator["kfd_node"].isdigit(): - raise TraceError("manifest accelerator KFD node is invalid") - if not isinstance(accelerator.get("gpu_id"), int) or accelerator["gpu_id"] <= 0: - raise TraceError("manifest accelerator GPU identity is invalid") + if self.manifest["runtime"] == "ds4": + required_paths.update({"runtime_checkout", "runner_executable", "runner_script", "exporter"}) + paths = self.manifest["paths"] + if not isinstance(paths, dict) or set(paths) != required_paths: + raise TraceError("manifest execution paths are invalid") + for label, value in paths.items(): + if not isinstance(label, str) or not isinstance(value, str) or not value.startswith("/"): + raise TraceError("manifest execution path is invalid") + if self.manifest["model"].get("path") != paths["model"]: + raise TraceError("manifest model path is not bound to execution paths") + if self.manifest["prompt"].get("path") != paths["prompt"]: + raise TraceError("manifest prompt path is not bound to execution paths") corpus_name = self.manifest["prompt"].get("corpus_name") if corpus_name not in CORPUS_SHA256: raise TraceError("prompt corpus is not in the fixed correctness corpus set") @@ -459,6 +877,7 @@ def _validate_manifest(self) -> None: provenance = self.manifest["prompt"].get("provenance") if not isinstance(provenance, dict): raise TraceError("prompt provenance reference is missing") + _require_exact_keys(provenance, {"path", "sha256"}, "prompt provenance reference") provenance_sha256 = provenance.get("sha256", "") if not isinstance(provenance_sha256, str) or re.fullmatch( r"[0-9a-f]{64}", provenance_sha256) is None: @@ -467,8 +886,8 @@ def _validate_manifest(self) -> None: raise TraceError("prompt provenance path is not content addressed") try: provenance_bytes = self._path(provenance["path"]).read_bytes() - provenance_record = json.loads(provenance_bytes.decode("ascii")) - except (OSError, UnicodeError, json.JSONDecodeError) as error: + provenance_record = strict_json_loads(provenance_bytes.decode("ascii")) + except (OSError, UnicodeError, TraceError) as error: raise TraceError(f"cannot read prompt provenance: {error}") from error if sha256_bytes(provenance_bytes) != provenance_sha256: raise TraceError("prompt provenance SHA-256 mismatch") @@ -489,12 +908,22 @@ def _validate_manifest(self) -> None: raise TraceError(f"prompt provenance {key} mismatch") if re.fullmatch(r"[0-9a-f]{64}", provenance_record.get("builder_sha256", "")) is None: raise TraceError("prompt provenance builder SHA-256 is invalid") + _require_exact_keys( + provenance_record, + set(provenance_checks) | {"builder_sha256"}, + "prompt provenance", + ) if self.manifest["prompt"].get("target_tokens") != expected_target: raise TraceError("prompt target token count does not fill the configured context") if self.manifest["runtime"] == "llama.cpp": candidate = self.manifest.get("candidate") if not isinstance(candidate, dict): raise TraceError("llama.cpp candidate attestation is missing") + _require_exact_keys( + candidate, + {"repository", "revision", "base_revision", "diff_sha256", "executable_sha256"}, + "llama.cpp candidate attestation", + ) if candidate.get("repository") != REPOSITORY: raise TraceError(f"candidate repository must be {REPOSITORY}") for key in ("revision", "base_revision", "diff_sha256", "executable_sha256"): @@ -523,10 +952,41 @@ def _validate_manifest(self) -> None: } if self.manifest["config"].get("deepseek41") != expected_config: raise TraceError("DeepSeek V4.1 configuration is invalid") - if self.manifest["comparison"].get("logits") != "byte-identical-f32": - raise TraceError("logit comparison policy must be byte-identical-f32") + expected_comparison = { + "tokens": "exact", + "engram_rows": "exact", + "expert_ids": "exact-original-id-space", + "expert_weights": "byte-identical-f32", + "attention_candidates": "exact", + "logits": "byte-identical-f32", + } + if self.manifest["comparison"] != expected_comparison: + raise TraceError("trace comparison policy is invalid") config = self.manifest["config"] if self.manifest["runtime"] == "llama.cpp": + _require_exact_keys( + config, + { + "context", + "batch", + "ubatch", + "device", + "device_architecture", + "device_pci_id", + "decode_steps", + "kv_type_k", + "kv_type_v", + "flash_attention", + "gpu_layers", + "load_mode", + "expert_cache_slots", + "expert_cache_bytes", + "tokenizer_add_bos", + "tokenizer_parse_special", + "deepseek41", + }, + "llama.cpp config", + ) if accelerator["backend_device"] != "ROCm0": raise TraceError("llama.cpp accelerator backend device must be ROCm0") if config.get("batch") != ADMITTED_BATCH or config.get("ubatch") != ADMITTED_UBATCH: @@ -540,27 +1000,55 @@ def _validate_manifest(self) -> None: config.get("device_pci_id") != accelerator["pci_device_id"]): raise TraceError("llama.cpp trace device identity is not bound to the accelerator attestation") if config.get("kv_type_k") != "f16" or config.get("kv_type_v") != "f16" or ( - config.get("flash_attention") not in (True, 1)) or config.get("load_mode") != 0: + config.get("flash_attention") is not True) or config.get("load_mode") != 0: raise TraceError("llama.cpp trace inference configuration is invalid") - if self.manifest["runtime"] == "ds4" and config.get("prefill_chunk") != ADMITTED_UBATCH: - raise TraceError("ds4 trace does not use the admitted prefill chunk") + if self.manifest["runtime"] == "ds4": + _require_exact_keys( + config, + { + "context", + "decode_steps", + "prefill_chunk", + "device_backend", + "device_registry_id", + "deepseek41", + }, + "ds4 config", + ) + if config.get("prefill_chunk") != ADMITTED_UBATCH: + raise TraceError("ds4 trace does not use the admitted prefill chunk") + if config.get("device_backend") != "Metal" or ( + config.get("device_registry_id") != accelerator["metal_registry_id"]): + raise TraceError("ds4 trace device identity is not bound to the accelerator attestation") for audit_phase in ("pre", "post"): phase_audits = self.manifest["audits"].get(audit_phase) if not isinstance(phase_audits, dict): raise TraceError(f"manifest {audit_phase} audit set is invalid") - for kind in ("memory", "swap", "watchdog"): + expected_kinds = ( + ("memory", "swap", "watchdog") + if self.manifest["runtime"] == "llama.cpp" + else ("memory", "swap", "runner") + ) + if set(phase_audits) != set(expected_kinds): + raise TraceError(f"manifest {audit_phase} audit kinds are invalid") + for kind in expected_kinds: self._validate_audit_reference(audit_phase, kind, phase_audits.get(kind)) def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: if not isinstance(audit, dict): raise TraceError(f"manifest {phase} {kind} audit reference is invalid") + _require_exact_keys( + audit, + {"path", "sha256", "created_unix"}, + f"manifest {phase} {kind} audit reference", + ) audit_path = audit.get("path") if not isinstance(audit_path, str) or not audit_path: raise TraceError(f"manifest {phase} {kind} audit path is invalid") digest = audit.get("sha256", "") if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError(f"manifest {phase} {kind} audit SHA-256 is invalid") - if not isinstance(audit.get("created_unix"), int) or audit["created_unix"] <= 0: + if type(audit.get("created_unix")) is not int or audit["created_unix"] <= 0: raise TraceError(f"manifest {phase} {kind} audit timestamp is invalid") expected_path = f"audits/{phase}/{digest}.json" if audit_path != expected_path: @@ -573,56 +1061,128 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: if sha256_bytes(evidence) != digest: raise TraceError(f"{phase} {kind} audit evidence SHA-256 mismatch") try: - record = json.loads(evidence.decode("ascii")) - except (UnicodeError, json.JSONDecodeError) as error: + record = strict_json_loads(evidence.decode("ascii")) + except (UnicodeError, TraceError) as error: raise TraceError(f"{phase} {kind} audit evidence is invalid: {error}") from error if record.get("kind") != kind or record.get("created_unix") != audit["created_unix"]: raise TraceError(f"{phase} {kind} audit evidence metadata mismatch") - if record.get("environment") != {"HIP_LAUNCH_BLOCKING": "1"}: + record_keys = {"created_unix", "kind", "environment", "data"} + if kind == "memory": + record_keys |= {"storage", "accelerator"} + if self.manifest["runtime"] == "ds4": + record_keys.add("host") + _require_exact_keys(record, record_keys, f"{phase} {kind} audit evidence") + expected_environment = ( + {"HIP_LAUNCH_BLOCKING": "1"} + if self.manifest["runtime"] == "llama.cpp" + else {} + ) + if record.get("environment") != expected_environment: raise TraceError(f"{phase} {kind} audit environment is invalid") if not isinstance(record.get("data"), dict): raise TraceError(f"{phase} {kind} audit evidence data is invalid") if kind == "memory": + _require_exact_keys( + record["data"], + {"mem_total_bytes", "mem_available_bytes", "mem_used_bytes"}, + f"{phase} memory audit data", + ) used = record["data"].get("mem_used_bytes") - if not isinstance(used, int) or used < 0 or used >= SOFT_MEMORY_LIMIT: + total = record["data"].get("mem_total_bytes") + available = record["data"].get("mem_available_bytes") + if type(used) is not int or used < 0: raise TraceError(f"{phase} memory audit evidence is invalid") + if self.manifest["runtime"] == "llama.cpp" and used >= SOFT_MEMORY_LIMIT: + raise TraceError(f"{phase} memory audit evidence is invalid") + if self.manifest["runtime"] == "ds4" and ( + type(total) is not int or total < 128 * 1024 * 1024 * 1024 or + type(available) is not int or available <= 0 or available > total or used > total): + raise TraceError(f"{phase} ds4 memory audit evidence is invalid") storage = record.get("storage") required_storage = { "model", "prompt", "output", "repository", "temporary_directory", } + if self.manifest["runtime"] == "ds4": + required_storage.update({"runtime_checkout", "runner_executable", "runner_script", "exporter"}) if not isinstance(storage, dict) or set(storage) != required_storage: raise TraceError(f"{phase} memory audit storage evidence is missing") for label, item in storage.items(): - if not isinstance(label, str) or not isinstance(item, dict): + if not isinstance(label, str): raise TraceError(f"{phase} memory audit storage evidence is invalid") - if item.get("rotational") is not False or not isinstance(item.get("nvme_device"), str): - raise TraceError(f"{phase} memory audit storage is not non-rotational NVMe") - if re.fullmatch(r"nvme[0-9]+(?:c[0-9]+)?n[0-9]+", item["nvme_device"]) is None: - raise TraceError(f"{phase} memory audit NVMe device identity is invalid") - for path_key in ("resolved_path", "existing_path", "mount_point", "block_device_path"): - value = item.get(path_key) - if not isinstance(value, str) or not value.startswith("/"): - raise TraceError(f"{phase} memory audit {path_key} is invalid") - if not isinstance(item.get("filesystem_type"), str) or not item["filesystem_type"]: - raise TraceError(f"{phase} memory audit filesystem type is invalid") - if not isinstance(item.get("mount_source"), str) or not item["mount_source"]: - raise TraceError(f"{phase} memory audit mount source is invalid") - if not item["mount_source"].startswith("/dev/"): - raise TraceError(f"{phase} memory audit mount source is not a local block device") - if re.fullmatch(r"[0-9]+:[0-9]+", item.get("device_number", "")) is None: - raise TraceError(f"{phase} memory audit device number is invalid") - if item["nvme_device"] not in Path(item["block_device_path"]).parts: - raise TraceError(f"{phase} memory audit block device ancestry is invalid") - try: - Path(item["resolved_path"]).relative_to(Path(item["mount_point"])) - Path(item["existing_path"]).relative_to(Path(item["mount_point"])) - except ValueError as error: - raise TraceError(f"{phase} memory audit mount ancestry is invalid") from error - if self.manifest["runtime"] == "llama.cpp" and record.get("accelerator") != self.manifest["accelerator"]: + validate_storage_attestation(self.manifest["runtime"], item) + if item["resolved_path"] != self.manifest["paths"][label]: + raise TraceError(f"{phase} memory audit {label} path differs from the manifest") + if record.get("accelerator") != self.manifest["accelerator"]: raise TraceError(f"{phase} memory audit accelerator evidence mismatch") + if self.manifest["runtime"] == "ds4": + host = validate_host_attestation(record.get("host")) + if host != self.manifest["host"]: + raise TraceError(f"{phase} ds4 host evidence mismatch") + if host["memory_bytes"] != total: + raise TraceError(f"{phase} ds4 host memory differs from the memory audit") if kind == "swap": - if record["data"].get("enabled") is not False or record["data"].get("entries") != []: - raise TraceError(f"{phase} swap audit evidence does not report zero configured swap") + if self.manifest["runtime"] == "llama.cpp": + _require_exact_keys(record["data"], {"enabled", "entries"}, f"{phase} swap audit data") + if record["data"].get("enabled") is not False or record["data"].get("entries") != []: + raise TraceError(f"{phase} swap audit evidence does not report zero configured swap") + elif record["data"] != { + "source": "darwin-sysctl-vm.swapusage", + "total_bytes": 0, + "used_bytes": 0, + "free_bytes": 0, + }: + raise TraceError(f"{phase} ds4 swap audit evidence does not report zero swap") + if kind == "runner": + data = record["data"] + _require_exact_keys( + data, + { + "format", + "version", + "runtime_kind", + "source", + "runner_pid", + "runner_parent_pid", + "runner_uid", + "runner_executable", + "runner_executable_sha256", + "runner_script", + "runner_script_sha256", + "exporter_path", + "exporter_sha256", + "checkout_path", + "checkout_revision", + "command_sha256", + }, + f"{phase} ds4 runner audit", + ) + expected = { + "format": "dsv41-runner-ownership", + "version": 1, + "runtime_kind": "apple-metal", + "source": "python-subprocess", + "exporter_sha256": self.manifest["build"]["sha256"], + "checkout_revision": DS4_REVISION, + "checkout_path": self.manifest["paths"]["runtime_checkout"], + "runner_executable": self.manifest["paths"]["runner_executable"], + "runner_script": self.manifest["paths"]["runner_script"], + "exporter_path": self.manifest["paths"]["exporter"], + } + for key, value in expected.items(): + if data.get(key) != value: + raise TraceError(f"{phase} ds4 runner {key} mismatch") + for key in ("runner_pid", "runner_parent_pid"): + if type(data.get(key)) is not int or data[key] <= 0: + raise TraceError(f"{phase} ds4 runner {key} is invalid") + if type(data.get("runner_uid")) is not int or data["runner_uid"] < 0: + raise TraceError(f"{phase} ds4 runner UID is invalid") + for key in ( + "runner_executable_sha256", + "runner_script_sha256", + "exporter_sha256", + "command_sha256"): + if re.fullmatch(r"[0-9a-f]{64}", data.get(key, "")) is None: + raise TraceError(f"{phase} ds4 runner {key} is invalid") if kind == "watchdog": required = ( "format", @@ -659,16 +1219,15 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "audit_fd", "audit", ) - if any(key not in record["data"] for key in required): - raise TraceError(f"{phase} watchdog audit evidence is incomplete") + _require_exact_keys(record["data"], set(required), f"{phase} watchdog audit evidence") data = record["data"] if data["format"] != WATCHDOG_LEASE_FORMAT or data["version"] != WATCHDOG_VERSION: raise TraceError(f"{phase} watchdog audit format is invalid") if not isinstance(data["lease_id"], str) or re.fullmatch(r"[0-9a-f]{32,64}", data["lease_id"]) is None: raise TraceError(f"{phase} watchdog audit lease ID is invalid") - if not isinstance(data["watchdog_pid"], int) or data["watchdog_pid"] <= 1: + if type(data["watchdog_pid"]) is not int or data["watchdog_pid"] <= 1: raise TraceError(f"{phase} watchdog audit PID is invalid") - if not isinstance(data["watchdog_start_time_ticks"], int) or data["watchdog_start_time_ticks"] <= 0: + if type(data["watchdog_start_time_ticks"]) is not int or data["watchdog_start_time_ticks"] <= 0: raise TraceError(f"{phase} watchdog audit start time is invalid") for key in ("watchdog_command_sha256", "watchdog_script_sha256"): if not isinstance(data[key], str) or re.fullmatch(r"[0-9a-f]{64}", data[key]) is None: @@ -688,12 +1247,12 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} watchdog timing policy is invalid") if data["procfs_root"] != "/proc": raise TraceError(f"{phase} watchdog procfs root is invalid") - if not isinstance(data["guardian_pid"], int) or data["guardian_pid"] <= 1 or ( - not isinstance(data["child_pid"], int) or data["child_pid"] <= 1) or ( - not isinstance(data["child_process_group_id"], int) or data["child_process_group_id"] <= 1): + if type(data["guardian_pid"]) is not int or data["guardian_pid"] <= 1 or ( + type(data["child_pid"]) is not int or data["child_pid"] <= 1) or ( + type(data["child_process_group_id"]) is not int or data["child_process_group_id"] <= 1): raise TraceError(f"{phase} watchdog child identity is invalid") for key in ("audit_device", "audit_inode", "audit_uid", "audit_fd"): - if not isinstance(data[key], int) or data[key] < 0: + if type(data[key]) is not int or data[key] < 0: raise TraceError(f"{phase} watchdog audit {key} is invalid") if data["audit_mode"] != 0o600: raise TraceError(f"{phase} watchdog audit mode is invalid") @@ -707,7 +1266,7 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: if not isinstance(data["watchdog_command_sha256"], str) or re.fullmatch( r"[0-9a-f]{64}", data["watchdog_command_sha256"]) is None: raise TraceError(f"{phase} watchdog audit command SHA-256 is invalid") - if not isinstance(data["heartbeat_unix"], int) or data["heartbeat_unix"] <= 0: + if type(data["heartbeat_unix"]) is not int or data["heartbeat_unix"] <= 0: raise TraceError(f"{phase} watchdog audit heartbeat timestamp is invalid") max_age = data.get("max_heartbeat_age_seconds") if not isinstance(max_age, (int, float)) or isinstance(max_age, bool) or ( @@ -719,12 +1278,17 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: audit_jsonl = data["audit"] if not isinstance(audit_jsonl, dict): raise TraceError(f"{phase} watchdog JSONL reference is invalid") + _require_exact_keys( + audit_jsonl, + {"path", "sha256", "event_count"}, + f"{phase} watchdog JSONL reference", + ) jsonl_digest = audit_jsonl.get("sha256", "") if not isinstance(jsonl_digest, str) or re.fullmatch(r"[0-9a-f]{64}", jsonl_digest) is None: raise TraceError(f"{phase} watchdog JSONL SHA-256 is invalid") if audit_jsonl.get("path") != f"audits/{phase}/{jsonl_digest}.jsonl": raise TraceError(f"{phase} watchdog JSONL path is invalid") - if not isinstance(audit_jsonl.get("event_count"), int) or audit_jsonl["event_count"] < 2: + if type(audit_jsonl.get("event_count")) is not int or audit_jsonl["event_count"] < 2: raise TraceError(f"{phase} watchdog JSONL event count is invalid") jsonl_path = self._path(audit_jsonl["path"]) try: @@ -737,8 +1301,8 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: if len(lines) != audit_jsonl["event_count"]: raise TraceError(f"{phase} watchdog JSONL event count mismatch") try: - events = [json.loads(line) for line in lines] - except json.JSONDecodeError as error: + events = [validate_watchdog_event(strict_json_loads(line)) for line in lines] + except TraceError as error: raise TraceError(f"{phase} watchdog JSONL is invalid: {error}") from error if not any(event.get("event") == "preflight" for event in events) or ( not any(event.get("event") == "child_started" for event in events)): @@ -769,12 +1333,17 @@ def _validate_coverage(self) -> None: expected = self.manifest.get("expected") if not isinstance(expected, dict): raise TraceError("manifest expected coverage is missing") + _require_exact_keys( + expected, + {"prompt_tokens", "decode_steps", "components"}, + "manifest expected coverage", + ) prompt_tokens = expected.get("prompt_tokens") decode_steps = expected.get("decode_steps") components = expected.get("components") - if not isinstance(prompt_tokens, int) or prompt_tokens <= 0: + if type(prompt_tokens) is not int or prompt_tokens <= 0: raise TraceError("expected prompt_tokens is invalid") - if not isinstance(decode_steps, int) or decode_steps <= 0: + if type(decode_steps) is not int or decode_steps <= 0: raise TraceError("expected decode_steps is invalid") if self.manifest.get("config", {}).get("decode_steps") != decode_steps: raise TraceError("expected decode_steps does not match config") @@ -827,7 +1396,8 @@ def _validate_coverage(self) -> None: decode_coverage = rules.get("decode") if input_coverage == "tokens": input_events = [event for event in events if event["phase"] == "input"] - if len(input_events) != 1 or input_events[0]["token_start"] != 0 or input_events[0]["token_count"] != prompt_tokens: + if len(input_events) != 1 or input_events[0]["token_start"] != 0 or ( + input_events[0]["token_count"] != prompt_tokens): raise TraceError(f"{component} input coverage is incomplete") for layer in expected_layers or [None]: layer_events = [event for event in events if event["layer"] == layer] @@ -952,8 +1522,6 @@ def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: checks = ( ("model.sha256", "model_identity"), ("model.architecture", "model_identity"), - ("accelerator.architecture", "accelerator_identity"), - ("accelerator.pci_device_id", "accelerator_identity"), ("prompt.sha256", "prompt_identity"), ("prompt.byte_count", "prompt_identity"), ("expected.prompt_tokens", "tokenizer"), From 0fb40432636bef9d257c3c440ffbb5d0f7b02ab9 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 02:27:21 -0700 Subject: [PATCH 22/56] trace : record session provenance Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 From 21f1908fd183a6353c91a5a50e7ecc009fe5b744 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 02:48:02 -0700 Subject: [PATCH 23/56] trace : close exporter attestation gaps Align native manifest field types with the v2 validator and harden runtime-specific storage, audit, and memory-resident state evidence. Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 199 ++++++++++++++++-- tools/deepseek-v41-trace/CMakeLists.txt | 5 + tools/deepseek-v41-trace/README.md | 4 +- tools/deepseek-v41-trace/host-attestation.h | 24 ++- tools/deepseek-v41-trace/llama-trace.cpp | 78 ++++++- tools/deepseek-v41-trace/preflight.py | 80 ++++--- tools/deepseek-v41-trace/run_ds4.py | 7 + tools/deepseek-v41-trace/run_matrix.py | 19 +- .../test-host-attestation.cpp | 27 ++- tools/deepseek-v41-trace/trace_format.py | 28 ++- 10 files changed, 402 insertions(+), 69 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 0bd366dbcd14..2b1fea0f3426 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -3,6 +3,8 @@ import importlib.util import io import json +import os +import subprocess import struct import sys import tempfile @@ -20,6 +22,7 @@ SPEC.loader.exec_module(trace) import run_llama import run_ds4 +import run_matrix import preflight import verify_ds4_anchors @@ -203,6 +206,7 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "environment": {"HIP_LAUNCH_BLOCKING": "1"}, "data": {"mem_total_bytes": 128, "mem_available_bytes": 64, "mem_used_bytes": 64}, "storage": STORAGE_ATTESTATION, + "storage_policy": json.loads(json.dumps(trace.NO_EXTERNAL_STATE_STORAGE)), "accelerator": dict(ACCELERATOR_ATTESTATION), }, "swap": { @@ -219,8 +223,10 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "format": trace.WATCHDOG_LEASE_FORMAT, "version": trace.WATCHDOG_VERSION, "lease_id": "1" * 32, + "state": "active", "lease_path": "/run/user/123/watchdog.lease", "watchdog_pid": 123, + "watchdog_start_time_utc": "1970-01-01T00:00:01.000Z", "watchdog_start_time_ticks": 456, "watchdog_command": "python3 /repo/scripts/strix_memory_watchdog.py", "watchdog_command_sha256": "7" * 64, @@ -268,6 +274,7 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "mem_used_bytes": 128 * 1024 * 1024 * 1024, }, "storage": DS4_STORAGE_ATTESTATION, + "storage_policy": json.loads(json.dumps(trace.NO_EXTERNAL_STATE_STORAGE)), "accelerator": dict(METAL_ACCELERATOR_ATTESTATION), "host": DS4_HOST_ATTESTATION, }, @@ -400,6 +407,7 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: label: record["resolved_path"] for label, record in storage.items() }, + "storage_policy": json.loads(json.dumps(trace.NO_EXTERNAL_STATE_STORAGE)), "comparison": { "tokens": "exact", "engram_rows": "exact", @@ -1049,7 +1057,7 @@ def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: def test_darwin_storage_and_host_preflight_are_fail_closed(self) -> None: with tempfile.TemporaryDirectory() as temp: - root = Path(temp) + root = Path(temp).resolve() for name in ("repo", "ds4", "tmp", "output"): (root / name).mkdir() model = root / "model.gguf" @@ -1125,7 +1133,8 @@ def command_text(*args: str) -> str: ({"SolidState": False}, "internal non-rotational"), ({"VolumeNetwork": True}, "local storage"), ({"DiskImage": True}, "local storage"), - ({"BusProtocol": "Network"}, "bus protocol")): + ({"BusProtocol": "Network"}, "NVMe-backed"), + ({"BusProtocol": "SATA"}, "NVMe-backed")): def invalid_info(path: Path, mutation: dict[str, object] = mutation) -> dict[str, object]: result = disk_info(path) result.update(mutation) @@ -1169,6 +1178,21 @@ def check_output(command, **_kwargs): self.assertEqual(result["_dsv41_mount_point"], "/System/Volumes/Data") self.assertEqual({key: value for key, value in result.items() if not key.startswith("_")}, disk_info) + def test_tmpdir_rejects_symlink_components(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + actual = root / "actual" + child = actual / "child" + child.mkdir(parents=True) + link = root / "link" + link.symlink_to(actual, target_is_directory=True) + for path in (link, Path(str(link) + "/"), link / ".", link / "child"): + with self.subTest(path=path), self.assertRaisesRegex( + preflight.PreflightError, "symlink"): + preflight.require_safe_tmpdir_path(path) + with self.assertRaisesRegex(preflight.PreflightError, "must not use /mnt/bigspace"): + preflight.require_safe_tmpdir_path(Path("/mnt/bigspace/escape")) + def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: with mock.patch.object( preflight, @@ -1184,7 +1208,8 @@ def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: busy_patterns=[], ) with tempfile.TemporaryDirectory() as temp: - missing = Path(temp) / "missing" + root = Path(temp).resolve() + missing = root / "missing" with mock.patch.dict( preflight.os.environ, {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(missing)}, @@ -1197,22 +1222,23 @@ def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: repo=Path("/home/repo"), busy_patterns=[], ) - actual = Path(temp) / "actual" - actual.mkdir() - link = Path(temp) / "link" + actual = root / "actual" + (actual / "child").mkdir(parents=True) + link = root / "link" link.symlink_to(actual, target_is_directory=True) - with mock.patch.dict( - preflight.os.environ, - {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(link)}, - clear=True): - with self.assertRaisesRegex(preflight.PreflightError, "must not be a symlink"): - preflight.run_strix_preflight( - model=Path("/home/model.gguf"), - prompt=Path("/home/prompt.txt"), - output=Path("/home/trace"), - repo=Path("/home/repo"), - busy_patterns=[], - ) + for path in (link, Path(str(link) + "/"), link / ".", link / "child"): + with self.subTest(path=path), mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(path)}, + clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "must not be a symlink"): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) def test_watchdog_lease_rejects_arbitrary_heartbeat_process(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -1591,6 +1617,18 @@ def test_rejects_ds4_accelerator_audit_removal(self) -> None: with self.assertRaisesRegex(trace.TraceError, "missing accelerator"): trace.TraceBundle(root) + def test_rejects_ds4_sata_storage_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["memory"])) + record["storage"]["model"]["bus_protocol"] = "SATA" + replace_audit_record(root, phase, "memory", record) + with self.assertRaisesRegex(trace.TraceError, "not NVMe-backed"): + trace.TraceBundle(root) + def test_rejects_cross_runtime_host_evidence(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" @@ -1687,6 +1725,131 @@ def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: with self.assertRaisesRegex(trace.TraceError, "environment is not macOS"): trace.TraceBundle(root) + def test_native_exporter_manifest_field_types_validate(self) -> None: + binary = Path(os.environ.get( + "DSV41_NATIVE_TRACE_BINARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "llama-deepseek-v41-trace", + )) + if not binary.is_file(): + self.skipTest("native trace exporter is not built") + command = [str(binary), "--dsv41-manifest-type-probe", "argument with space"] + result = subprocess.run(command, check=True, capture_output=True, text=True) + native = trace.strict_json_loads(result.stdout) + self.assertIsInstance(native["environment"]["command"], str) + self.assertEqual(json.loads(native["environment"]["command"]), command) + self.assertIs(native["config"]["flash_attention"], True) + self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("llama.cpp") + if sys.platform.startswith("linux"): + trace_manifest["environment"]["system_info"] = native["environment"]["system_info"] + trace_manifest["environment"]["command"] = native["environment"]["command"] + trace_manifest["config"]["flash_attention"] = native["config"]["flash_attention"] + trace_manifest["storage_policy"] = native["storage_policy"] + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + trace.TraceBundle(root) + + def test_prompt_builder_result_becomes_strict_provenance(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + builder = root / "prompt-builder" + model = root / "model.gguf" + corpus = root / "corpus.txt" + output = root / "prompt.txt" + tmpdir = root / "tmp" + builder.write_bytes(b"builder") + model.write_bytes(b"model") + corpus.write_bytes(b"corpus") + tmpdir.mkdir() + + def run_builder(command, **_kwargs): + output.write_bytes(b"prompt") + return subprocess.CompletedProcess( + command, + 0, + json.dumps({ + "target_tokens": 2, + "actual_tokens": 2, + "byte_count": 6, + "add_bos": True, + "temporary_directory": str(tmpdir.resolve()), + }), + "", + ) + + with mock.patch.dict(os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( + run_matrix.subprocess, "run", side_effect=run_builder), mock.patch.object( + sys, "stderr", io.StringIO()): + result = run_matrix.prepare_prompt( + builder=builder, + model=model, + corpus=corpus, + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=output, + target_tokens=2, + ) + provenance_path = Path(result["provenance_path"]) + provenance = trace.strict_json_loads(provenance_path.read_text(encoding="ascii")) + self.assertEqual( + set(provenance), + { + "format", "version", "corpus_name", "corpus_sha256", "model_sha256", + "prompt_sha256", "prompt_byte_count", "builder_sha256", "target_tokens", "actual_tokens", + }, + ) + preflight.validate_prompt_provenance( + provenance_path, + prompt=output, + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + model_sha256=trace.MODEL_SHA256, + target_tokens=2, + path_resolver=lambda path, _label: path.resolve(), + ) + + def test_rejects_cross_runtime_and_unknown_audit_envelopes(self) -> None: + for mutation, message in ( + (lambda value: value["audits"].update({"unknown": {}}), "audit envelope"), + (lambda value: value["audits"]["pre"].update({"watchdog": {}}), "audit kinds")): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + mutation(trace_manifest) + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_requires_no_external_cache_or_state_storage(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["storage_policy"]["external_cache_paths"] = ["/Users/oracle/cache"] + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "storage policy"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["memory"])) + del record["storage_policy"] + replace_audit_record(root, "pre", "memory", record) + with self.assertRaisesRegex(trace.TraceError, "storage_policy"): + trace.TraceBundle(root) + + def test_accepts_authentic_watchdog_lease_fields(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + trace.TraceBundle(root) + def test_rejects_boolean_accelerator_identities(self) -> None: for runtime, field in ( ("llama.cpp", "gpu_id"), diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index 104ee2c4b854..ff447cde482b 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -14,6 +14,11 @@ if(LLAMA_TOOLS_INSTALL) endif() if(LLAMA_BUILD_TESTS) + if(TEST test-deepseek41-trace) + set_property( + TEST test-deepseek41-trace + APPEND PROPERTY ENVIRONMENT "DSV41_NATIVE_TRACE_BINARY=$") + endif() set(HOST_TEST_TARGET test-deepseek41-trace-host) add_executable(${HOST_TEST_TARGET} test-host-attestation.cpp) target_link_libraries(${HOST_TEST_TARGET} PRIVATE ggml) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 766d5b13c04b..2671f1244820 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -10,6 +10,8 @@ Each trace is a directory: - `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. - `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. +Every manifest and memory audit declares that the expert cache and KV cache are memory-resident and that there are no external cache or state paths. Missing, substituted, or additional file-backed cache/state declarations fail closed. + The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. Every bundle carries one strict runtime-discriminated accelerator attestation. A llama.cpp candidate uses the `strix-rocm` kind: the selected backend device must map through its PCI identity and Linux KFD topology to `gfx_target_version=110501` (`gfx1151`). A ds4 oracle uses the `apple-metal` kind: the selected Metal device records its registry ID, reported architecture, unified-memory property, and recommended working-set size. Missing kinds, unknown kinds, cross-kind fields, duplicate JSON keys, and mixed evidence fail closed. Cross-runtime comparison does not require the two physical accelerators or PCI identities to match; each runtime proves its own execution environment, while the model, prompt, inference semantics, and complete output artifacts remain exact comparison inputs. @@ -121,7 +123,7 @@ Use `run_llama.py` on the validation host instead of calling the exporter direct ## Apple Metal oracle execution gate -The external ds4 exporter is not present in the pinned canonical checkout. It remains blocked until a separately built executable is reviewed and its exact SHA-256 is added to the otherwise empty `APPROVED_EXPORTERS` map. Approval is checked before the exporter can run, including device-only preflight. `run_ds4.py` requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state local storage. Network volumes, disk images, external/non-internal devices, non-solid-state media, incomplete device identity, and lexical or resolved forbidden paths fail closed. +The external ds4 exporter is not present in the pinned canonical checkout. It remains blocked until a separately built executable is reviewed and its exact SHA-256 is added to the otherwise empty `APPROVED_EXPORTERS` map. Approval is checked before the exporter can run, including device-only preflight. `run_ds4.py` requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/host-attestation.h b/tools/deepseek-v41-trace/host-attestation.h index a55dd3aa903e..1f621237d048 100644 --- a/tools/deepseek-v41-trace/host-attestation.h +++ b/tools/deepseek-v41-trace/host-attestation.h @@ -104,10 +104,27 @@ static inline fs::path existing_ancestor(const fs::path & path) { return fs::canonical(current); } -static inline void require_usable_directory(const fs::path & path, const char * label) { - if (fs::is_symlink(path)) { - throw std::runtime_error(std::string(label) + " must not be a symlink"); +static inline void reject_symlink_components(const fs::path & path, const char * label) { + fs::path current; + for (const fs::path & part : fs::absolute(path)) { + current /= part; + std::error_code error; + const fs::file_status status = fs::symlink_status(current, error); + if (!error && fs::is_symlink(status)) { + throw std::runtime_error(std::string(label) + " must not be a symlink or contain symlink components"); + } + if (error == std::errc::no_such_file_or_directory) { + break; + } + if (error) { + throw std::runtime_error( + std::string(label) + " symlink status cannot be read: " + error.message()); + } } +} + +static inline void require_usable_directory(const fs::path & path, const char * label) { + reject_symlink_components(path, label); if (!fs::is_directory(path)) { throw std::runtime_error(std::string(label) + " must be an existing directory"); } @@ -132,6 +149,7 @@ static inline storage_attestation require_nvme_path( if (path_is_within(absolute, forbidden)) { throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); } + reject_symlink_components(path, label); const fs::path resolved = fs::weakly_canonical(absolute); const fs::path existing = existing_ancestor(resolved); if (path_is_within(resolved, forbidden)) { diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 5ad7144de315..51eb32973ab5 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -41,6 +41,10 @@ extern "C" { #include #endif +#if !defined(_WIN32) +#include +#endif + namespace fs = std::filesystem; using json = nlohmann::ordered_json; @@ -340,6 +344,12 @@ static void validate_watchdog(const json & data) { } #endif +static void bind_memory_audit_metadata(json & result, const json & audit) { + result["accelerator"] = audit.value("accelerator", json::object()); + result["storage"] = audit.value("storage", json::object()); + result["storage_policy"] = audit.value("storage_policy", json::object()); +} + static json audit_reference(const char * environment_name, const char * expected_kind) { const fs::path path = required_environment(environment_name); dsv41::require_nvme_path(path, "audit"); @@ -377,8 +387,7 @@ static json audit_reference(const char * environment_name, const char * expected if (std::string(expected_kind) == "watchdog") { result["data"] = audit["data"]; } else if (std::string(expected_kind) == "memory") { - result["accelerator"] = audit.value("accelerator", json::object()); - result["storage"] = audit.value("storage", json::object()); + bind_memory_audit_metadata(result, audit); } return result; } @@ -620,6 +629,28 @@ static std::vector command_line(int argc, char ** argv) { return result; } +static std::string command_line_json(int argc, char ** argv) { + return json(command_line(argc, argv)).dump(); +} + +static bool flash_attention_enabled(enum llama_flash_attn_type value) { + return value == LLAMA_FLASH_ATTN_TYPE_ENABLED; +} + +static std::string runtime_system_info(const common_params & params) { +#if defined(_WIN32) + const std::string platform = "Windows"; +#else + struct utsname info = {}; + if (uname(&info) != 0) { + throw std::runtime_error("cannot query operating system identity"); + } + const std::string platform = + std::string(info.sysname) + " " + info.release + " " + info.machine; +#endif + return platform + "; " + common_params_get_system_info(params); +} + static json accelerator_json(const dsv41::accelerator_attestation & accelerator) { return { {"format", "dsv41-accelerator-attestation"}, @@ -658,9 +689,42 @@ static json storage_json(const dsv41::storage_attestation & storage) { }; } +static json storage_policy_json() { + return { + {"format", "dsv41-state-storage-policy"}, + {"version", 1}, + {"expert_cache", "memory-resident"}, + {"kv_cache", "memory-resident"}, + {"external_cache_paths", json::array()}, + {"external_state_paths", json::array()}, + }; +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { + if (argc >= 2 && std::string(argv[1]) == "--dsv41-manifest-type-probe") { + common_params params; + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + json probe = { + {"environment", { + {"system_info", runtime_system_info(params)}, + {"command", command_line_json(argc, argv)}, + }}, + {"config", { + {"flash_attention", flash_attention_enabled(params.flash_attn_type)}, + }}, + }; + json audit_reference_probe; + bind_memory_audit_metadata(audit_reference_probe, { + {"accelerator", json::object()}, + {"storage", json::object()}, + {"storage_policy", storage_policy_json()}, + }); + probe["storage_policy"] = audit_reference_probe["storage_policy"]; + std::cout << probe.dump() << '\n'; + return 0; + } if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { common_init(); ggml_backend_load_all(); @@ -712,6 +776,9 @@ int main(int argc, char ** argv) { if (memory_audit.value("accelerator", json::object()) != accelerator_json(configured_accelerator)) { throw std::runtime_error("preflight accelerator audit does not match the selected execution device"); } + if (memory_audit.value("storage_policy", json::object()) != storage_policy_json()) { + throw std::runtime_error("preflight external cache/state storage policy is invalid"); + } const json audited_storage = memory_audit.value("storage", json::object()); if (audited_storage.value("model", json::object()) != storage_json(model_storage) || audited_storage.value("prompt", json::object()) != storage_json(prompt_storage) || @@ -803,6 +870,7 @@ int main(int argc, char ** argv) { {"repository", audited_storage["repository"].value("resolved_path", "")}, {"temporary_directory", temporary_storage.resolved_path.string()}, }}, + {"storage_policy", storage_policy_json()}, {"config", { {"context", llama_n_ctx(ctx)}, {"batch", params.n_batch}, @@ -813,7 +881,7 @@ int main(int argc, char ** argv) { {"decode_steps", params.n_predict}, {"kv_type_k", ggml_type_name(params.cache_type_k)}, {"kv_type_v", ggml_type_name(params.cache_type_v)}, - {"flash_attention", static_cast(params.flash_attn_type)}, + {"flash_attention", flash_attention_enabled(params.flash_attn_type)}, {"gpu_layers", params.n_gpu_layers}, {"load_mode", static_cast(params.load_mode)}, {"expert_cache_slots", params.expert_cache_slots}, @@ -845,8 +913,8 @@ int main(int argc, char ** argv) { {"logits", "byte-identical-f32"}, }}, {"environment", { - {"system_info", common_params_get_system_info(params)}, - {"command", command_line(argc, argv)}, + {"system_info", runtime_system_info(params)}, + {"command", command_line_json(argc, argv)}, }}, {"audits", { {"memory", memory_audit}, diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index 95c846a4aba8..523a9804504a 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Callable -from trace_format import TraceError, validate_watchdog_event +from trace_format import NO_EXTERNAL_STATE_STORAGE, TraceError, validate_watchdog_event FORBIDDEN_ROOT = Path("/mnt/bigspace") SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 @@ -58,6 +58,35 @@ def resolved(path: Path) -> Path: return path.expanduser().resolve() +def require_no_symlink_components(path: Path, label: str) -> Path: + absolute = path.expanduser().absolute() + current = Path(absolute.anchor) + for part in absolute.parts[1:]: + current /= part + if current.is_symlink(): + raise PreflightError(f"{label} must not be a symlink or contain symlink components") + if not current.exists(): + break + return absolute + + +def reject_forbidden_path( + path: Path, + label: str, + forbidden_root: Path = FORBIDDEN_ROOT) -> Path: + absolute = path.expanduser().absolute() + try: + absolute.relative_to(forbidden_root) + except ValueError: + return absolute + raise PreflightError(f"{label} must not use /mnt/bigspace: {absolute}") + + +def require_safe_tmpdir_path(path: Path) -> Path: + lexical_path = reject_forbidden_path(path, "TMPDIR") + return require_no_symlink_components(lexical_path, "TMPDIR") + + def _decode_mount_field(value: str) -> str: return re.sub( r"\\([0-7]{3})", @@ -83,13 +112,7 @@ def storage_attestation( sys_dev_block_root: Path = Path("/sys/dev/block"), sys_class_block_root: Path = Path("/sys/class/block"), forbidden_root: Path = FORBIDDEN_ROOT) -> dict[str, object]: - lexical_path = path.expanduser().absolute() - try: - lexical_path.relative_to(forbidden_root) - except ValueError: - pass - else: - raise PreflightError(f"{label} must not use /mnt/bigspace: {lexical_path}") + lexical_path = reject_forbidden_path(path, label, forbidden_root) path = resolved(lexical_path) try: path.relative_to(forbidden_root) @@ -230,13 +253,7 @@ def darwin_storage_attestation( *, disk_info: Callable[[Path], dict[str, object]] = _diskutil_info, forbidden_root: Path = FORBIDDEN_ROOT) -> dict[str, object]: - lexical_path = path.expanduser().absolute() - try: - lexical_path.relative_to(forbidden_root) - except ValueError: - pass - else: - raise PreflightError(f"{label} must not use /mnt/bigspace: {lexical_path}") + lexical_path = reject_forbidden_path(path, label, forbidden_root) path = resolved(lexical_path) try: path.relative_to(forbidden_root) @@ -265,8 +282,8 @@ def darwin_storage_attestation( raise PreflightError(f"{label} {name} cannot be resolved: {path}") if not mount_point.startswith("/"): raise PreflightError(f"{label} mount point is invalid: {mount_point}") - if bus_protocol.lower() in {"network", "virtual", "disk image"}: - raise PreflightError(f"{label} bus protocol is not local: {bus_protocol}") + if bus_protocol.lower() not in {"nvme", "apple fabric"}: + raise PreflightError(f"{label} storage is not NVMe-backed: {bus_protocol}") try: filesystem_device = os.stat(existing).st_dev if filesystem_device != os.stat(mount_point).st_dev: @@ -971,13 +988,11 @@ def run_strix_preflight( tmpdir_value = os.environ.get("TMPDIR") if not tmpdir_value: raise PreflightError("TMPDIR is required for NVMe-only correctness runs") - tmpdir_input = Path(tmpdir_value).expanduser().absolute() - if tmpdir_input.is_symlink(): - raise PreflightError("TMPDIR must not be a symlink") - tmpdir = resolved(tmpdir_input) + tmpdir_input = require_safe_tmpdir_path(Path(tmpdir_value)) + tmp_storage = storage_attestation(tmpdir_input, "temporary directory") + tmpdir = _attested_resolved_path(tmp_storage, "temporary directory") if not tmpdir.is_dir() or not os.access(tmpdir, os.W_OK | os.X_OK): raise PreflightError("TMPDIR must be an existing writable directory") - tmp_storage = storage_attestation(tmpdir, "temporary directory") model = _attested_resolved_path(model_storage, "model") prompt = _attested_resolved_path(prompt_storage, "prompt") output = _attested_resolved_path(output_storage, "trace output") @@ -1005,6 +1020,7 @@ def run_strix_preflight( "watchdog": watchdog, "active_workloads": [], "environment": {"HIP_LAUNCH_BLOCKING": "1"}, + "storage_policy": dict(NO_EXTERNAL_STATE_STORAGE), "storage": { "model": model_storage, "prompt": prompt_storage, @@ -1044,13 +1060,11 @@ def run_oracle_preflight( tmpdir_value = os.environ.get("TMPDIR") if not tmpdir_value: raise PreflightError("TMPDIR is required for ds4 oracle correctness runs") - tmpdir_input = Path(tmpdir_value).expanduser().absolute() - if tmpdir_input.is_symlink(): - raise PreflightError("TMPDIR must not be a symlink") - tmpdir = resolved(tmpdir_input) + tmpdir_input = require_safe_tmpdir_path(Path(tmpdir_value)) + tmp_storage = darwin_storage_attestation(tmpdir_input, "temporary directory", disk_info=disk_info) + tmpdir = _attested_resolved_path(tmp_storage, "temporary directory") if not tmpdir.is_dir() or not os.access(tmpdir, os.W_OK | os.X_OK): raise PreflightError("TMPDIR must be an existing writable directory") - tmp_storage = darwin_storage_attestation(tmpdir, "temporary directory", disk_info=disk_info) model = _attested_resolved_path(model_storage, "model") prompt = _attested_resolved_path(prompt_storage, "prompt") output = _attested_resolved_path(output_storage, "trace output") @@ -1103,6 +1117,7 @@ def run_oracle_preflight( "accelerator": accelerator, "active_workloads": [], "environment": {}, + "storage_policy": dict(NO_EXTERNAL_STATE_STORAGE), "storage": { "model": model_storage, "prompt": prompt_storage, @@ -1147,13 +1162,10 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: events = [] for line_number, line in enumerate(audit_text.splitlines(), start=1): try: - event = strict_json_loads(line) - except json.JSONDecodeError as error: + event = validate_watchdog_event(strict_json_loads(line)) + except (PreflightError, TraceError) as error: raise PreflightError( f"watchdog audit line {line_number} is invalid while snapshotting: {error}") from error - if not isinstance(event, dict) or not isinstance(event.get("event"), str): - raise PreflightError( - f"watchdog audit line {line_number} is not an event while snapshotting") events.append(event) if not events: raise PreflightError("watchdog audit snapshot is empty") @@ -1171,6 +1183,7 @@ def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: } if key == "memory": value["storage"] = audit["storage"] + value["storage_policy"] = audit["storage_policy"] value["accelerator"] = audit["accelerator"] if "host" in audit: value["host"] = audit["host"] @@ -1322,6 +1335,9 @@ def validate_prompt_provenance( builder_sha256 = record.get("builder_sha256", "") if not isinstance(builder_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", builder_sha256) is None: raise PreflightError("prompt provenance builder SHA-256 is invalid") + required_keys = set(expected) | {"builder_sha256"} + if set(record) != required_keys: + raise PreflightError("prompt provenance fields are invalid") return {"path": str(path), "bytes": data, "record": record} diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index ee242d736151..2fe59435c73a 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -24,6 +24,7 @@ ADMITTED_UBATCH, CORPUS_SHA256, MODEL_SHA256, + NO_EXTERNAL_STATE_STORAGE, TraceBundle, TraceError, canonical_json, @@ -185,6 +186,12 @@ def bind_oracle_attestation( if "paths" in manifest and manifest["paths"] != paths: raise PreflightError("ds4 trace execution paths differ from preflight") manifest["paths"] = paths + storage_policy = audit.get("storage_policy") + if storage_policy != NO_EXTERNAL_STATE_STORAGE: + raise PreflightError("ds4 external cache/state storage policy is invalid") + if "storage_policy" in manifest and manifest["storage_policy"] != storage_policy: + raise PreflightError("ds4 trace external cache/state storage policy differs from preflight") + manifest["storage_policy"] = storage_policy host = audit.get("host") if not isinstance(host, dict): raise PreflightError("ds4 host attestation is missing") diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index fcae024f95c7..d97eb2983dd7 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -58,16 +58,23 @@ def prepare_prompt( if result.returncode != 0: raise RuntimeError(f"prompt builder failed: {result.stderr.strip()}") try: - record = strict_json_loads(result.stdout) + native_record = strict_json_loads(result.stdout) except TraceError as error: raise RuntimeError(f"prompt builder returned invalid JSON: {error}") from error - if record.get("actual_tokens") != target_tokens: + if not isinstance(native_record, dict) or set(native_record) != { + "target_tokens", "actual_tokens", "byte_count", "add_bos", "temporary_directory"}: + raise RuntimeError("prompt builder returned an invalid result schema") + if native_record.get("target_tokens") != target_tokens or native_record.get("actual_tokens") != target_tokens: raise RuntimeError("prompt builder did not produce the requested token count") - temporary_directory = record.pop("temporary_directory", None) + if type(native_record.get("byte_count")) is not int or native_record["byte_count"] != output.stat().st_size: + raise RuntimeError("prompt builder byte count does not match its output") + if type(native_record.get("add_bos")) is not bool: + raise RuntimeError("prompt builder add_bos result is invalid") + temporary_directory = native_record["temporary_directory"] expected_temporary_directory = os.environ.get("TMPDIR") if not expected_temporary_directory or temporary_directory != str(resolved(Path(expected_temporary_directory))): raise RuntimeError("prompt builder did not attest the selected temporary directory") - record.update({ + record = { "format": "dsv41-prompt-provenance", "version": 1, "corpus_name": corpus_name, @@ -76,7 +83,9 @@ def prepare_prompt( "prompt_sha256": sha256_file(output), "prompt_byte_count": output.stat().st_size, "builder_sha256": sha256_file(builder), - }) + "target_tokens": target_tokens, + "actual_tokens": target_tokens, + } provenance_path = output.with_suffix(output.suffix + ".provenance.json") provenance_path.write_text( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", diff --git a/tools/deepseek-v41-trace/test-host-attestation.cpp b/tools/deepseek-v41-trace/test-host-attestation.cpp index 8459ce9d9281..4676df557ddc 100644 --- a/tools/deepseek-v41-trace/test-host-attestation.cpp +++ b/tools/deepseek-v41-trace/test-host-attestation.cpp @@ -32,7 +32,7 @@ static void require_failure(Function function, const std::string & expected) { } int main() { - const fs::path root = fs::temp_directory_path() / + const fs::path root = fs::canonical(fs::temp_directory_path()) / ("dsv41-host-attestation-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); try { @@ -65,6 +65,22 @@ int main() { dsv41::require_usable_directory(root / "tmp-link", "TMPDIR"); }, "must not be a symlink"); + require_failure( + [&]() { + dsv41::require_usable_directory(fs::path((root / "tmp-link").string() + "/"), "TMPDIR"); + }, + "must not be a symlink"); + require_failure( + [&]() { + dsv41::require_usable_directory(root / "tmp-link" / ".", "TMPDIR"); + }, + "must not be a symlink"); + fs::create_directories(btrfs / "child"); + require_failure( + [&]() { + dsv41::require_usable_directory(root / "tmp-link" / "child", "TMPDIR"); + }, + "must not be a symlink"); const fs::path sys = root / "sys"; const fs::path nvme1 = sys / "devices" / "pci" / "block" / "nvme1n1"; @@ -107,6 +123,13 @@ int main() { btrfs_attestation.nvme_device != "nvme0n1") { throw std::runtime_error("btrfs NVMe partition attestation mismatch"); } + require_failure( + [&]() { + dsv41::require_nvme_path( + root / "tmp-link" / "child", "symlink traversal", mountinfo, + sys / "dev" / "block", sys / "class" / "block"); + }, + "must not be a symlink"); require_failure( [&]() { dsv41::require_nvme_path( @@ -137,7 +160,7 @@ int main() { xfs / "escape" / "data", "symlink escape", mountinfo, sys / "dev" / "block", sys / "class" / "block"); }, - "local block device"); + "must not be a symlink"); require_failure( [&]() { dsv41::require_nvme_path( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 4906f7bbda8e..da59ed2f38bd 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -25,6 +25,14 @@ WATCHDOG_REVISION = "778db6f50eae04e6c232c69b9575bdbd0747962b" WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f" APPROVED_WATCHDOGS = {WATCHDOG_SCRIPT_SHA256: WATCHDOG_REVISION} +NO_EXTERNAL_STATE_STORAGE = { + "format": "dsv41-state-storage-policy", + "version": 1, + "expert_cache": "memory-resident", + "kv_cache": "memory-resident", + "external_cache_paths": [], + "external_state_paths": [], +} ADMITTED_UBATCH = 32 ADMITTED_BATCH = 2048 EXPERT_COUNT = 384 @@ -328,8 +336,8 @@ def validate_storage_attestation(runtime: str, item: Any) -> dict[str, Any]: for key in ("device_identifier", "parent_whole_disk", "bus_protocol"): if not isinstance(item.get(key), str) or not item[key]: raise TraceError(f"ds4 storage {key} is invalid") - if item["bus_protocol"].lower() in {"network", "virtual", "disk image"}: - raise TraceError("ds4 storage bus protocol is not local") + if item["bus_protocol"].lower() not in {"nvme", "apple fabric"}: + raise TraceError("ds4 storage is not NVMe-backed") if type(item.get("filesystem_device")) is not int or item["filesystem_device"] < 0: raise TraceError("ds4 storage filesystem device identity is invalid") else: @@ -766,6 +774,7 @@ def _validate_manifest(self) -> None: "comparison", "environment", "paths", + "storage_policy", "audits", "expected", } @@ -846,6 +855,8 @@ def _validate_manifest(self) -> None: raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") if self.manifest["model"].get("architecture") != "deepseek41": raise TraceError("model architecture must be deepseek41") + if self.manifest["storage_policy"] != NO_EXTERNAL_STATE_STORAGE: + raise TraceError("manifest external cache/state storage policy is invalid") accelerator = validate_accelerator_attestation(self.manifest["runtime"], self.manifest["accelerator"]) if self.manifest["runtime"] == "ds4": if "host" not in self.manifest: @@ -1020,6 +1031,7 @@ def _validate_manifest(self) -> None: if config.get("device_backend") != "Metal" or ( config.get("device_registry_id") != accelerator["metal_registry_id"]): raise TraceError("ds4 trace device identity is not bound to the accelerator attestation") + _require_exact_keys(self.manifest["audits"], {"pre", "post"}, "manifest audit envelope") for audit_phase in ("pre", "post"): phase_audits = self.manifest["audits"].get(audit_phase) if not isinstance(phase_audits, dict): @@ -1068,7 +1080,7 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} {kind} audit evidence metadata mismatch") record_keys = {"created_unix", "kind", "environment", "data"} if kind == "memory": - record_keys |= {"storage", "accelerator"} + record_keys |= {"storage", "storage_policy", "accelerator"} if self.manifest["runtime"] == "ds4": record_keys.add("host") _require_exact_keys(record, record_keys, f"{phase} {kind} audit evidence") @@ -1099,6 +1111,8 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: type(available) is not int or available <= 0 or available > total or used > total): raise TraceError(f"{phase} ds4 memory audit evidence is invalid") storage = record.get("storage") + if record.get("storage_policy") != self.manifest["storage_policy"]: + raise TraceError(f"{phase} memory audit storage policy mismatch") required_storage = { "model", "prompt", "output", "repository", "temporary_directory", } @@ -1188,8 +1202,10 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "format", "version", "lease_id", + "state", "lease_path", "watchdog_pid", + "watchdog_start_time_utc", "watchdog_start_time_ticks", "watchdog_command", "watchdog_command_sha256", @@ -1223,12 +1239,18 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: data = record["data"] if data["format"] != WATCHDOG_LEASE_FORMAT or data["version"] != WATCHDOG_VERSION: raise TraceError(f"{phase} watchdog audit format is invalid") + if data["state"] != "active": + raise TraceError(f"{phase} watchdog audit state is invalid") if not isinstance(data["lease_id"], str) or re.fullmatch(r"[0-9a-f]{32,64}", data["lease_id"]) is None: raise TraceError(f"{phase} watchdog audit lease ID is invalid") if type(data["watchdog_pid"]) is not int or data["watchdog_pid"] <= 1: raise TraceError(f"{phase} watchdog audit PID is invalid") if type(data["watchdog_start_time_ticks"]) is not int or data["watchdog_start_time_ticks"] <= 0: raise TraceError(f"{phase} watchdog audit start time is invalid") + if not isinstance(data["watchdog_start_time_utc"], str) or re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z", + data["watchdog_start_time_utc"]) is None: + raise TraceError(f"{phase} watchdog audit UTC start time is invalid") for key in ("watchdog_command_sha256", "watchdog_script_sha256"): if not isinstance(data[key], str) or re.fullmatch(r"[0-9a-f]{64}", data[key]) is None: raise TraceError(f"{phase} watchdog audit {key} is invalid") From e1732bc8e9e29d4398586001100b6849db19fdc7 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 02:56:32 -0700 Subject: [PATCH 24/56] trace : validate lexical temporary directory Reject an inherited TMPDIR unless its original lexical pathname is an existing writable directory before resolution and NVMe attestation. Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 51 +++++++++++++++++++++++++++ tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/preflight.py | 12 ++++++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 2b1fea0f3426..9d837d488d21 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -1193,6 +1193,57 @@ def test_tmpdir_rejects_symlink_components(self) -> None: with self.assertRaisesRegex(preflight.PreflightError, "must not use /mnt/bigspace"): preflight.require_safe_tmpdir_path(Path("/mnt/bigspace/escape")) + def test_full_preflights_reject_unusable_lexical_tmpdir(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + (root / "good" / "tmp").mkdir(parents=True) + unusable = root / "good" / "missing" / ".." / "tmp" + actual = root / "actual" + actual.mkdir() + link = root / "link" + link.symlink_to(actual, target_is_directory=True) + self.assertFalse(unusable.is_dir()) + self.assertFalse(os.access(unusable, os.W_OK | os.X_OK)) + self.assertTrue(link.is_symlink()) + cases = ( + (unusable, "original lexical path"), + (link, "symlink"), + ) + for tmpdir, message in cases: + with self.subTest(runtime="strix-rocm", tmpdir=tmpdir), mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(tmpdir)}, + clear=True), mock.patch.object( + preflight, + "storage_attestation", + return_value=storage_record("/home/test")): + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + with self.subTest(runtime="apple-metal", tmpdir=tmpdir), mock.patch.dict( + preflight.os.environ, + {"TMPDIR": str(tmpdir)}, + clear=True), mock.patch.object( + preflight, + "darwin_storage_attestation", + return_value=metal_storage_record("/Users/oracle/test")): + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.run_oracle_preflight( + model=Path("/Users/oracle/model.gguf"), + prompt=Path("/Users/oracle/prompt.txt"), + output=Path("/Users/oracle/trace"), + repo=Path("/Users/oracle/repo"), + checkout=Path("/Users/oracle/ds4"), + busy_patterns=[], + accelerator={}, + runner={}, + ) + def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: with mock.patch.object( preflight, diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 2671f1244820..23d428205434 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -85,7 +85,7 @@ The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd07479 The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. -Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory, must be an existing writable non-symlink directory on verified NVMe, and has no `/tmp` fallback. These metadata commands do not execute the model: +Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory, its original lexical pathname must be an existing writable non-symlink directory on verified NVMe, and there is no `/tmp` fallback. These metadata commands do not execute the model: ```sh MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index 523a9804504a..637b155a2ab6 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -7,6 +7,7 @@ import platform import plistlib import re +import stat import subprocess import sys import time @@ -84,7 +85,16 @@ def reject_forbidden_path( def require_safe_tmpdir_path(path: Path) -> Path: lexical_path = reject_forbidden_path(path, "TMPDIR") - return require_no_symlink_components(lexical_path, "TMPDIR") + require_no_symlink_components(lexical_path, "TMPDIR") + try: + status = os.lstat(lexical_path) + except OSError as error: + raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") from error + if not stat.S_ISDIR(status.st_mode): + raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") + if not os.access(lexical_path, os.W_OK | os.X_OK): + raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") + return lexical_path def _decode_mount_field(value: str) -> str: From 5694bfada5478174f997d9f9256cf1be25fa25c4 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 03:15:26 -0700 Subject: [PATCH 25/56] trace : bind canonical watchdog artifacts Assisted-by: GPT-5.6 Sol Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 295 +++++++++++++++++++++-- tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/llama-trace.cpp | 49 ++-- tools/deepseek-v41-trace/preflight.py | 90 ++++--- tools/deepseek-v41-trace/trace_format.py | 26 +- 5 files changed, 387 insertions(+), 75 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 9d837d488d21..74c663029949 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import copy import importlib.util import io import json @@ -9,9 +10,10 @@ import sys import tempfile import unittest -from unittest import mock from argparse import Namespace +from datetime import datetime, timezone from pathlib import Path +from unittest import mock TRACE_DIR = Path(__file__).parents[1] / "tools" / "deepseek-v41-trace" sys.path.insert(0, str(TRACE_DIR)) @@ -224,6 +226,10 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "version": trace.WATCHDOG_VERSION, "lease_id": "1" * 32, "state": "active", + "file_device": 1, + "file_inode": 2, + "file_uid": 1000, + "file_mode": 0o600, "lease_path": "/run/user/123/watchdog.lease", "watchdog_pid": 123, "watchdog_start_time_utc": "1970-01-01T00:00:01.000Z", @@ -254,6 +260,7 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "audit_uid": 1000, "audit_mode": 0o600, "audit_fd": 3, + "audit_sha256": WATCHDOG_JSONL_SHA256, "audit": { "path": "", "sha256": WATCHDOG_JSONL_SHA256, @@ -324,6 +331,26 @@ def replace_audit_record(root: Path, phase: str, kind: str, record: dict[str, ob ) +def replace_watchdog_events(root: Path, phase: str, events: list[dict[str, object]]) -> None: + records = [] + for event in events: + record = json.loads(json.dumps(event)) + records.append(record) + data = b"".join( + (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + for record in records + ) + digest = trace.sha256_bytes(data) + path = root / "audits" / phase / f"{digest}.jsonl" + path.write_bytes(data) + record = json.loads(json.dumps(AUDIT_RECORDS["watchdog"])) + record["data"]["audit_sha256"] = digest + record["data"]["audit"]["path"] = f"audits/{phase}/{digest}.jsonl" + record["data"]["audit"]["sha256"] = digest + record["data"]["audit"]["event_count"] = len(records) + replace_audit_record(root, phase, "watchdog", record) + + def provenance_bytes(prompt: bytes = b"abc") -> bytes: record = { "format": "dsv41-prompt-provenance", @@ -1202,21 +1229,41 @@ def test_full_preflights_reject_unusable_lexical_tmpdir(self) -> None: actual.mkdir() link = root / "link" link.symlink_to(actual, target_is_directory=True) + home = root / "home" + (home / "tmp").mkdir(parents=True) + literal_home = Path("~/tmp") self.assertFalse(unusable.is_dir()) self.assertFalse(os.access(unusable, os.W_OK | os.X_OK)) + self.assertFalse(literal_home.is_dir()) + self.assertFalse(os.access(literal_home, os.W_OK | os.X_OK)) self.assertTrue(link.is_symlink()) cases = ( (unusable, "original lexical path"), (link, "symlink"), + (literal_home, "existing writable directory"), ) + + def strix_storage(_path: Path, label: str) -> dict[str, object]: + if label == "temporary directory": + raise AssertionError("unusable TMPDIR reached storage attestation") + return storage_record("/home/test") + + def oracle_storage( + _path: Path, + label: str, + **_kwargs: object) -> dict[str, object]: + if label == "temporary directory": + raise AssertionError("unusable TMPDIR reached storage attestation") + return metal_storage_record("/Users/oracle/test") + for tmpdir, message in cases: with self.subTest(runtime="strix-rocm", tmpdir=tmpdir), mock.patch.dict( preflight.os.environ, - {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(tmpdir)}, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(tmpdir), "HOME": str(home)}, clear=True), mock.patch.object( preflight, "storage_attestation", - return_value=storage_record("/home/test")): + side_effect=strix_storage): with self.assertRaisesRegex(preflight.PreflightError, message): preflight.run_strix_preflight( model=Path("/home/model.gguf"), @@ -1227,11 +1274,11 @@ def test_full_preflights_reject_unusable_lexical_tmpdir(self) -> None: ) with self.subTest(runtime="apple-metal", tmpdir=tmpdir), mock.patch.dict( preflight.os.environ, - {"TMPDIR": str(tmpdir)}, + {"TMPDIR": str(tmpdir), "HOME": str(home)}, clear=True), mock.patch.object( preflight, "darwin_storage_attestation", - return_value=metal_storage_record("/Users/oracle/test")): + side_effect=oracle_storage): with self.assertRaisesRegex(preflight.PreflightError, message): preflight.run_oracle_preflight( model=Path("/Users/oracle/model.gguf"), @@ -1496,6 +1543,163 @@ def test_approved_watchdog_is_exact(self) -> None: self.assertEqual(preflight.WATCHDOG_VERSION, 2) self.assertEqual(trace.WATCHDOG_VERSION, 2) + def test_canonical_watchdog_artifacts_embed_and_validate(self) -> None: + revision = preflight.WATCHDOG_REVISION + repository = Path(__file__).parents[1] + source = subprocess.check_output( + ["git", "show", f"{revision}:scripts/strix_memory_watchdog.py"], + cwd=repository, + ) + self.assertEqual(preflight.sha256_bytes(source), preflight.WATCHDOG_SCRIPT_SHA256) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + repo = root / "repo" + script = repo / "scripts" / "strix_memory_watchdog.py" + script.parent.mkdir(parents=True) + script.write_bytes(source) + watchdog = preflight._load_watchdog_module(script) + + lease_path = root / "watchdog.lease" + heartbeat_path = root / "watchdog.heartbeat" + audit_path = root / "watchdog.jsonl" + child_command = ["python3", "run_matrix.py"] + arguments = [ + "--soft-gib", "116", + "--emergency-gib", "118", + "--grace-seconds", "30", + "--sample-interval-seconds", "1", + "--heartbeat-max-age-seconds", "5", + "--lease-path", str(lease_path), + "--heartbeat-path", str(heartbeat_path), + "--audit-path", str(audit_path), + "--", + *child_command, + ] + config = watchdog.parse_args(arguments) + paths = config.validate() + now = datetime.now(timezone.utc).replace(microsecond=0) + monotonic_ns = 1_000_000_000 + procfs = root / "proc" + watchdog_pid = os.getpid() + guardian_pid = watchdog_pid + 100000 + child_pid = guardian_pid + 1 + + def proc_stat(pid: int, parent: int, group: int, start: int) -> str: + fields = ["S", str(parent), str(group), *(["0"] * 16), str(start)] + return f"{pid} (test) " + " ".join(fields) + "\n" + + for pid in (watchdog_pid, guardian_pid, child_pid): + (procfs / str(pid)).mkdir(parents=True) + (procfs / str(watchdog_pid) / "stat").write_text( + proc_stat(watchdog_pid, 1, watchdog_pid, 1000), encoding="ascii") + (procfs / str(guardian_pid) / "stat").write_text( + proc_stat(guardian_pid, watchdog_pid, guardian_pid, 2000), encoding="ascii") + (procfs / str(child_pid) / "stat").write_text( + proc_stat(child_pid, guardian_pid, guardian_pid, 3000), encoding="ascii") + watchdog_argv = [sys.executable, str(script), *arguments] + watchdog_cmdline = b"\0".join(os.fsencode(value) for value in watchdog_argv) + b"\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(watchdog_cmdline) + (procfs / str(watchdog_pid) / "exe").symlink_to(Path(sys.executable).resolve()) + (procfs / str(watchdog_pid) / "cwd").symlink_to(repo) + + class FakeProcess: + pid = guardian_pid + + @staticmethod + def poll() -> None: + return None + + @staticmethod + def wait(timeout: float | None = None) -> int: + del timeout + return 0 + + guardian = watchdog.GuardianProcess(FakeProcess(), child_pid, -1) + snapshot = watchdog.HostSnapshot( + 128 * 1024 * 1024 * 1024, + 64 * 1024 * 1024 * 1024, + (), + ) + logger = watchdog.AuditLogger(io.StringIO(), wall_clock=lambda: now) + logger.open_persistent(audit_path) + state = watchdog._state_fields( + snapshot, snapshot.used_bytes, None, None, "not_created", "none") + logger.emit( + "preflight", + **state, + soft_bytes=config.soft_bytes, + emergency_bytes=config.emergency_bytes, + strict_ceiling_bytes=watchdog.STRICT_CEILING_BYTES, + ) + manager = watchdog.LeaseManager( + config, + paths, + process_procfs_root=procfs, + wall_clock=lambda: now, + monotonic_ns=lambda: monotonic_ns, + ) + manager.start(guardian, logger) + logger.lease_manager = manager + state = watchdog._state_fields( + snapshot, snapshot.used_bytes, guardian, None, "active", "none") + logger.emit("child_started", **state, command=child_command) + logger.heartbeat(state) + fd_path = procfs / str(watchdog_pid) / "fd" + fd_path.mkdir() + (fd_path / str(logger.persistent_identity()["fd"])).symlink_to(audit_path) + + validated = watchdog.validate_active_lease( + lease_path, + expected_script_path=script, + expected_executable_path=Path(sys.executable), + expected_soft_bytes=trace.SOFT_MEMORY_LIMIT, + expected_emergency_bytes=trace.WATCHDOG_EMERGENCY_LIMIT, + expected_procfs_root=Path("/proc"), + expected_command=child_command, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=5.0, + current_process_id=child_pid, + process_procfs_root=procfs, + monotonic_ns=lambda: monotonic_ns, + pidfd_open=lambda _: os.open("/dev/null", os.O_RDONLY), + ) + watchdog_audit = preflight._watchdog_audit_result( + validated, + watchdog_revision=revision, + lease_path=lease_path, + heartbeat_path=heartbeat_path, + audit_path=audit_path, + audit_event_count=2, + procfs_root=procfs, + ) + created_unix = int(now.timestamp()) + audit = { + "created_unix": created_unix, + "runtime_kind": "strix-rocm", + "memory": copy.deepcopy(AUDIT_RECORDS["memory"]["data"]), + "swap": copy.deepcopy(AUDIT_RECORDS["swap"]["data"]), + "watchdog": watchdog_audit, + "environment": copy.deepcopy(AUDIT_RECORDS["memory"]["environment"]), + "storage": copy.deepcopy(STORAGE_ATTESTATION), + "storage_policy": copy.deepcopy(trace.NO_EXTERNAL_STATE_STORAGE), + "accelerator": copy.deepcopy(ACCELERATOR_ATTESTATION), + } + trace_root = root / "trace" + with trace.TraceBundleWriter(trace_root, manifest("llama.cpp")) as writer: + add_required_events(writer) + audit_sets = {} + for phase in ("pre", "post"): + paths = preflight.write_audits(root / f"{phase}-audits", audit) + preflight.seal_audits(paths) + audit_sets[phase] = paths + preflight.bind_embedded_audits(trace_root, audit_sets) + try: + trace.TraceBundle(trace_root) + finally: + logger.close() + def test_workload_scan_ignores_guarded_process_ancestry(self) -> None: with tempfile.TemporaryDirectory() as temp: procfs = Path(temp) @@ -1783,23 +1987,18 @@ def test_native_exporter_manifest_field_types_validate(self) -> None: )) if not binary.is_file(): self.skipTest("native trace exporter is not built") - command = [str(binary), "--dsv41-manifest-type-probe", "argument with space"] - result = subprocess.run(command, check=True, capture_output=True, text=True) - native = trace.strict_json_loads(result.stdout) - self.assertIsInstance(native["environment"]["command"], str) - self.assertEqual(json.loads(native["environment"]["command"]), command) - self.assertIs(native["config"]["flash_attention"], True) - self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) with tempfile.TemporaryDirectory() as temp: - root = Path(temp) / "trace" - trace_manifest = manifest("llama.cpp") - if sys.platform.startswith("linux"): - trace_manifest["environment"]["system_info"] = native["environment"]["system_info"] - trace_manifest["environment"]["command"] = native["environment"]["command"] - trace_manifest["config"]["flash_attention"] = native["config"]["flash_attention"] - trace_manifest["storage_policy"] = native["storage_policy"] - with trace.TraceBundleWriter(root, trace_manifest) as writer: + root = Path(temp) / "trace with space" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + command = [str(binary), "--dsv41-manifest-type-probe", str(manifest_path)] + subprocess.run(command, check=True) + native = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + self.assertIsInstance(native["environment"]["command"], str) + self.assertEqual(json.loads(native["environment"]["command"]), command) + self.assertIs(native["config"]["flash_attention"], True) + self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) trace.TraceBundle(root) def test_prompt_builder_result_becomes_strict_provenance(self) -> None: @@ -1901,6 +2100,61 @@ def test_accepts_authentic_watchdog_lease_fields(self) -> None: add_required_events(writer) trace.TraceBundle(root) + for field, value, message in ( + ("file_inode", True, "file_inode is invalid"), + ("file_mode", 0o644, "file mode is invalid"), + ("audit_sha256", "f" * 64, "live and embedded audit SHA-256 differ")): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = copy.deepcopy(AUDIT_RECORDS["watchdog"]) + record["data"]["audit"]["path"] = ( + f"audits/{phase}/{WATCHDOG_JSONL_SHA256}.jsonl") + record["data"][field] = value + replace_audit_record(root, phase, "watchdog", record) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_enforces_watchdog_final_error_classification(self) -> None: + terminal = { + key: copy.deepcopy(value) + for key, value in WATCHDOG_EVENTS[1].items() + if key not in {"command", "event_id", "parent_event_sha256", "record_sha256"} + } + terminal.update({ + "event": "final", + "classification": "internal_error", + "exit_code": 1, + "error": "test internal error", + }) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, terminal]) + trace.TraceBundle(root) + + for classification, error in ( + ("internal_error", None), + ("child_exit", "fabricated error")): + with self.subTest(classification=classification), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + with self.assertRaisesRegex(trace.TraceError, "error presence does not match"): + trace.TraceBundle(root) + def test_rejects_boolean_accelerator_identities(self) -> None: for runtime, field in ( ("llama.cpp", "gpu_id"), @@ -1931,6 +2185,7 @@ def test_rejects_unknown_watchdog_event_fields(self) -> None: jsonl = root / "audits" / phase / f"{digest}.jsonl" jsonl.write_bytes(data) record = json.loads(json.dumps(AUDIT_RECORDS["watchdog"])) + record["data"]["audit_sha256"] = digest record["data"]["audit"]["path"] = f"audits/{phase}/{digest}.jsonl" record["data"]["audit"]["sha256"] = digest record["data"]["audit"]["event_count"] = len(events) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 23d428205434..ea81e7aab07c 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -85,7 +85,7 @@ The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd07479 The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. -Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory, its original lexical pathname must be an existing writable non-symlink directory on verified NVMe, and there is no `/tmp` fallback. These metadata commands do not execute the model: +Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory, its original lexical pathname must be an existing writable non-symlink directory on verified NVMe, and there is no `/tmp` fallback. Literal shell shorthand such as `~/tmp` is rejected because environment values are not shell-expanded for the launched process. These metadata commands do not execute the model: ```sh MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 51eb32973ab5..eb74d980a4ed 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -700,29 +700,40 @@ static json storage_policy_json() { }; } +static void write_manifest_type_probe(const fs::path & path, int argc, char ** argv) { + const std::vector bytes = read_file(path); + json manifest = json::parse(bytes.begin(), bytes.end()); + if (!manifest.is_object()) { + throw std::runtime_error("manifest type probe input is not a JSON object"); + } + common_params params; + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; +#if defined(__linux__) + manifest["environment"]["system_info"] = runtime_system_info(params); +#endif + manifest["environment"]["command"] = command_line_json(argc, argv); + manifest["config"]["flash_attention"] = flash_attention_enabled(params.flash_attn_type); + manifest["storage_policy"] = storage_policy_json(); + + const fs::path temp = path.string() + ".tmp"; + { + std::ofstream stream(temp, std::ios::binary | std::ios::trunc); + stream << manifest.dump() << '\n'; + if (!stream) { + throw std::runtime_error("cannot write manifest type probe"); + } + } + fs::rename(temp, path); +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { if (argc >= 2 && std::string(argv[1]) == "--dsv41-manifest-type-probe") { - common_params params; - params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; - json probe = { - {"environment", { - {"system_info", runtime_system_info(params)}, - {"command", command_line_json(argc, argv)}, - }}, - {"config", { - {"flash_attention", flash_attention_enabled(params.flash_attn_type)}, - }}, - }; - json audit_reference_probe; - bind_memory_audit_metadata(audit_reference_probe, { - {"accelerator", json::object()}, - {"storage", json::object()}, - {"storage_policy", storage_policy_json()}, - }); - probe["storage_policy"] = audit_reference_probe["storage_policy"]; - std::cout << probe.dump() << '\n'; + if (argc != 3) { + throw std::runtime_error("--dsv41-manifest-type-probe requires a manifest path"); + } + write_manifest_type_probe(argv[2], argc, argv); return 0; } if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index 637b155a2ab6..81a0591e1a62 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -85,15 +85,17 @@ def reject_forbidden_path( def require_safe_tmpdir_path(path: Path) -> Path: lexical_path = reject_forbidden_path(path, "TMPDIR") - require_no_symlink_components(lexical_path, "TMPDIR") try: - status = os.lstat(lexical_path) + status = os.lstat(path) except OSError as error: raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") from error + if stat.S_ISLNK(status.st_mode): + raise PreflightError("TMPDIR must not be a symlink or contain symlink components") if not stat.S_ISDIR(status.st_mode): raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") - if not os.access(lexical_path, os.W_OK | os.X_OK): + if not os.access(path, os.W_OK | os.X_OK): raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") + require_no_symlink_components(lexical_path, "TMPDIR") return lexical_path @@ -621,6 +623,49 @@ def _load_watchdog_module(script: Path) -> object: return module +def _watchdog_audit_result( + lease: dict[str, object], + *, + watchdog_revision: str, + lease_path: Path, + heartbeat_path: Path, + audit_path: Path, + audit_event_count: int, + procfs_root: Path = Path("/proc")) -> dict[str, object]: + try: + heartbeat_record = strict_json_loads(heartbeat_path.read_text(encoding="ascii")) + updated = datetime.fromisoformat(str(heartbeat_record["updated_at"]).replace("Z", "+00:00")) + heartbeat_unix = int(updated.timestamp()) + except (OSError, UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + raise PreflightError(f"canonical watchdog heartbeat is invalid: {error}") from error + try: + audit_sha256 = sha256_bytes(audit_path.read_bytes()) + except OSError as error: + raise PreflightError(f"cannot hash canonical watchdog audit: {error}") from error + watchdog_command = lease.get("watchdog_command") + if not isinstance(watchdog_command, str) or not watchdog_command: + try: + command_bytes = ( + procfs_root / str(lease["watchdog_pid"]) / "cmdline").read_bytes() + except (OSError, KeyError) as error: + raise PreflightError(f"cannot read canonical watchdog command: {error}") from error + watchdog_command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() + if not watchdog_command: + raise PreflightError("canonical watchdog command is empty") + result = dict(lease) + result.update({ + "watchdog_revision": watchdog_revision, + "lease_path": str(lease_path), + "watchdog_command": watchdog_command, + "heartbeat_unix": heartbeat_unix, + "audit_live_path": str(audit_path), + "audit_path": str(audit_path), + "audit_sha256": audit_sha256, + "audit_event_count": audit_event_count, + }) + return result + + def _canonical_watchdog_audit( repo: Path, *, @@ -689,37 +734,14 @@ def _canonical_watchdog_audit( monotonic=monotonic, sleeper=sleeper, ) - try: - heartbeat_record = strict_json_loads(heartbeat_path.read_text(encoding="ascii")) - updated = datetime.fromisoformat(str(heartbeat_record["updated_at"]).replace("Z", "+00:00")) - heartbeat_unix = int(updated.timestamp()) - except (OSError, UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: - raise PreflightError(f"canonical watchdog heartbeat is invalid: {error}") from error - try: - audit_sha256 = sha256_bytes(audit_path.read_bytes()) - except OSError as error: - raise PreflightError(f"cannot hash canonical watchdog audit: {error}") from error - watchdog_command = lease.get("watchdog_command") - if not isinstance(watchdog_command, str) or not watchdog_command: - try: - command_bytes = ( - Path("/proc") / str(lease["watchdog_pid"]) / "cmdline").read_bytes() - except (OSError, KeyError) as error: - raise PreflightError(f"cannot read canonical watchdog command: {error}") from error - watchdog_command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() - if not watchdog_command: - raise PreflightError("canonical watchdog command is empty") - result = dict(lease) - result.update({ - "watchdog_revision": watchdog_revision, - "lease_path": str(lease_path), - "watchdog_command": watchdog_command, - "heartbeat_unix": heartbeat_unix, - "audit_path": str(audit_path), - "audit_sha256": audit_sha256, - "audit_event_count": len(events), - }) - return result + return _watchdog_audit_result( + lease, + watchdog_revision=watchdog_revision, + lease_path=lease_path, + heartbeat_path=heartbeat_path, + audit_path=audit_path, + audit_event_count=len(events), + ) def watchdog_audit( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index da59ed2f38bd..d3c20aed4355 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -406,6 +406,15 @@ def validate_host_attestation(host: Any) -> dict[str, Any]: "process_group_status", "threshold_reason", } +WATCHDOG_ERROR_CLASSIFICATIONS = { + "configuration_error", + "internal_error", + "launch_error", + "lease_error", + "procfs_error", + "signal_error", + "termination_timeout", +} def validate_watchdog_event(event: Any) -> dict[str, Any]: @@ -485,7 +494,10 @@ def validate_watchdog_event(event: Any) -> dict[str, Any]: raise TraceError("watchdog JSONL classification is invalid") if type(event["exit_code"]) is not int: raise TraceError("watchdog JSONL exit code is invalid") - if "error" in event and (not isinstance(event["error"], str) or not event["error"]): + requires_error = event["classification"] in WATCHDOG_ERROR_CLASSIFICATIONS + if ("error" in event) != requires_error: + raise TraceError("watchdog JSONL error presence does not match classification") + if requires_error and (not isinstance(event["error"], str) or not event["error"]): raise TraceError("watchdog JSONL error is invalid") secondary_errors = event.get("secondary_errors") if secondary_errors is not None: @@ -1203,6 +1215,10 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "version", "lease_id", "state", + "file_device", + "file_inode", + "file_uid", + "file_mode", "lease_path", "watchdog_pid", "watchdog_start_time_utc", @@ -1233,6 +1249,7 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "audit_uid", "audit_mode", "audit_fd", + "audit_sha256", "audit", ) _require_exact_keys(record["data"], set(required), f"{phase} watchdog audit evidence") @@ -1241,6 +1258,11 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} watchdog audit format is invalid") if data["state"] != "active": raise TraceError(f"{phase} watchdog audit state is invalid") + for key in ("file_device", "file_inode", "file_uid"): + if type(data[key]) is not int or data[key] < 0: + raise TraceError(f"{phase} watchdog lease {key} is invalid") + if data["file_mode"] != 0o600: + raise TraceError(f"{phase} watchdog lease file mode is invalid") if not isinstance(data["lease_id"], str) or re.fullmatch(r"[0-9a-f]{32,64}", data["lease_id"]) is None: raise TraceError(f"{phase} watchdog audit lease ID is invalid") if type(data["watchdog_pid"]) is not int or data["watchdog_pid"] <= 1: @@ -1308,6 +1330,8 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: jsonl_digest = audit_jsonl.get("sha256", "") if not isinstance(jsonl_digest, str) or re.fullmatch(r"[0-9a-f]{64}", jsonl_digest) is None: raise TraceError(f"{phase} watchdog JSONL SHA-256 is invalid") + if data["audit_sha256"] != jsonl_digest: + raise TraceError(f"{phase} watchdog live and embedded audit SHA-256 differ") if audit_jsonl.get("path") != f"audits/{phase}/{jsonl_digest}.jsonl": raise TraceError(f"{phase} watchdog JSONL path is invalid") if type(audit_jsonl.get("event_count")) is not int or audit_jsonl["event_count"] < 2: From de8879f702a3cd1f53a97b761e6ec707336c5ac9 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 03:26:21 -0700 Subject: [PATCH 26/56] trace : align canonical terminal schema Assisted-by: GPT-5.6 Sol Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 154 +++++++++++++++++++++-- tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/preflight.py | 2 + tools/deepseek-v41-trace/trace_format.py | 18 ++- 4 files changed, 158 insertions(+), 18 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 74c663029949..9abd27845222 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -1231,17 +1231,9 @@ def test_full_preflights_reject_unusable_lexical_tmpdir(self) -> None: link.symlink_to(actual, target_is_directory=True) home = root / "home" (home / "tmp").mkdir(parents=True) - literal_home = Path("~/tmp") self.assertFalse(unusable.is_dir()) self.assertFalse(os.access(unusable, os.W_OK | os.X_OK)) - self.assertFalse(literal_home.is_dir()) - self.assertFalse(os.access(literal_home, os.W_OK | os.X_OK)) self.assertTrue(link.is_symlink()) - cases = ( - (unusable, "original lexical path"), - (link, "symlink"), - (literal_home, "existing writable directory"), - ) def strix_storage(_path: Path, label: str) -> dict[str, object]: if label == "temporary directory": @@ -1256,8 +1248,8 @@ def oracle_storage( raise AssertionError("unusable TMPDIR reached storage attestation") return metal_storage_record("/Users/oracle/test") - for tmpdir, message in cases: - with self.subTest(runtime="strix-rocm", tmpdir=tmpdir), mock.patch.dict( + def assert_rejected(tmpdir: Path, message: str) -> None: + with mock.patch.dict( preflight.os.environ, {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(tmpdir), "HOME": str(home)}, clear=True), mock.patch.object( @@ -1272,7 +1264,7 @@ def oracle_storage( repo=Path("/home/repo"), busy_patterns=[], ) - with self.subTest(runtime="apple-metal", tmpdir=tmpdir), mock.patch.dict( + with mock.patch.dict( preflight.os.environ, {"TMPDIR": str(tmpdir), "HOME": str(home)}, clear=True), mock.patch.object( @@ -1291,6 +1283,60 @@ def oracle_storage( runner={}, ) + for tmpdir, message in ( + (unusable, "original lexical path"), + (link, "symlink")): + with self.subTest(tmpdir=tmpdir): + assert_rejected(tmpdir, message) + + literal_parent = root / "~" + literal_target = root / "literal" + (literal_target / "tmp").mkdir(parents=True) + literal_parent.symlink_to(literal_target, target_is_directory=True) + previous_cwd = Path.cwd() + try: + os.chdir(root) + literal_home = Path("~/tmp") + self.assertTrue(literal_home.is_dir()) + self.assertTrue(os.access(literal_home, os.W_OK | os.X_OK)) + self.assertNotEqual(literal_home.absolute(), literal_home.expanduser().absolute()) + assert_rejected(literal_home, "absolute literal path") + finally: + os.chdir(previous_cwd) + + model = root / "model.gguf" + prompt = root / "prompt.txt" + output = root / "output" + repo = root / "repo" + model.write_bytes(b"model") + prompt.write_bytes(b"prompt") + output.mkdir() + repo.mkdir() + valid_tmpdir = root / "good" / "tmp" + + def valid_strix_storage(path: Path, _label: str) -> dict[str, object]: + return storage_record(str(path.resolve())) + + with mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(valid_tmpdir)}, + clear=True), mock.patch.object( + preflight, "storage_attestation", side_effect=valid_strix_storage), mock.patch.object( + preflight, "swap_audit", return_value={"enabled": False, "entries": []}), mock.patch.object( + preflight, "watchdog_audit", return_value=copy.deepcopy( + AUDIT_RECORDS["watchdog"]["data"])), mock.patch.object( + preflight, "matching_workloads", return_value=[]), mock.patch.object( + preflight, "memory_audit", return_value=copy.deepcopy( + AUDIT_RECORDS["memory"]["data"])): + result = preflight.run_strix_preflight( + model=model, + prompt=prompt, + output=output, + repo=repo, + busy_patterns=[], + ) + self.assertEqual(result["storage"]["temporary_directory"]["resolved_path"], str(valid_tmpdir)) + def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: with mock.patch.object( preflight, @@ -1621,6 +1667,54 @@ def wait(timeout: float | None = None) -> int: 64 * 1024 * 1024 * 1024, (), ) + canonical_finals = {} + for classification, exit_code in ( + ("procfs_error", watchdog.EXIT_PROCFS_ERROR), + ("signal_error", watchdog.EXIT_SIGNAL_ERROR)): + stream = io.StringIO() + final_logger = watchdog.AuditLogger(stream, wall_clock=lambda: now) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._kill_and_finish( + final_logger, + guardian, + snapshot, + snapshot.used_bytes, + classification, + exit_code, + f"test {classification}", + lambda _pid, _signal: "sigkill_sent", + ) + final_event = trace.strict_json_loads(stream.getvalue().splitlines()[-1]) + self.assertNotIn("error", final_event) + trace.validate_watchdog_event(final_event) + canonical_finals[classification] = final_event + + class TimeoutProcess(FakeProcess): + @staticmethod + def wait(timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(child_command, timeout) + + timeout_stream = io.StringIO() + timeout_logger = watchdog.AuditLogger(timeout_stream, wall_clock=lambda: now) + timeout_guardian = watchdog.GuardianProcess(TimeoutProcess(), child_pid, -1) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._kill_and_finish( + timeout_logger, + timeout_guardian, + snapshot, + snapshot.used_bytes, + "procfs_error", + watchdog.EXIT_PROCFS_ERROR, + "test timeout", + lambda _pid, _signal: "sigkill_sent", + ) + timeout_final = trace.strict_json_loads(timeout_stream.getvalue().splitlines()[-1]) + self.assertEqual(timeout_final["classification"], "termination_timeout") + self.assertEqual(timeout_final["process_group_status"], "sigkill_timeout") + self.assertIsInstance(timeout_final["error"], str) + trace.validate_watchdog_event(timeout_final) + canonical_finals["termination_timeout"] = timeout_final + logger = watchdog.AuditLogger(io.StringIO(), wall_clock=lambda: now) logger.open_persistent(audit_path) state = watchdog._state_fields( @@ -1697,6 +1791,14 @@ def wait(timeout: float | None = None) -> int: preflight.bind_embedded_audits(trace_root, audit_sets) try: trace.TraceBundle(trace_root) + for classification, final_event in canonical_finals.items(): + final_root = root / f"trace-{classification}" + with trace.TraceBundleWriter(final_root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events( + final_root, phase, [*WATCHDOG_EVENTS, final_event]) + trace.TraceBundle(final_root) finally: logger.close() @@ -2118,6 +2220,17 @@ def test_accepts_authentic_watchdog_lease_fields(self) -> None: trace.TraceBundle(root) def test_enforces_watchdog_final_error_classification(self) -> None: + self.assertEqual(trace.WATCHDOG_REQUIRED_ERROR_CLASSIFICATIONS, { + "configuration_error", + "internal_error", + "launch_error", + "lease_error", + "termination_timeout", + }) + self.assertEqual(trace.WATCHDOG_OPTIONAL_ERROR_CLASSIFICATIONS, { + "procfs_error", + "signal_error", + }) terminal = { key: copy.deepcopy(value) for key, value in WATCHDOG_EVENTS[1].items() @@ -2137,6 +2250,25 @@ def test_enforces_watchdog_final_error_classification(self) -> None: replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, terminal]) trace.TraceBundle(root) + for classification, error in ( + ("procfs_error", None), + ("procfs_error", "initial snapshot failed"), + ("signal_error", None), + ("signal_error", "cannot signal process group")): + with self.subTest(classification=classification, error=error), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + trace.TraceBundle(root) + for classification, error in ( ("internal_error", None), ("child_exit", "fabricated error")): diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index ea81e7aab07c..3000cf80dcf7 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -85,7 +85,7 @@ The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd07479 The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. -Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory, its original lexical pathname must be an existing writable non-symlink directory on verified NVMe, and there is no `/tmp` fallback. Literal shell shorthand such as `~/tmp` is rejected because environment values are not shell-expanded for the launched process. These metadata commands do not execute the model: +Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory and must be an absolute literal pathname to an existing writable non-symlink directory on verified NVMe; there is no `/tmp` fallback. Relative paths and shell shorthand such as `~/tmp` are rejected because environment values are not shell-expanded for the launched process. These metadata commands do not execute the model: ```sh MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index 81a0591e1a62..c3ed40a6120f 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -84,6 +84,8 @@ def reject_forbidden_path( def require_safe_tmpdir_path(path: Path) -> Path: + if not path.is_absolute(): + raise PreflightError("TMPDIR must use an absolute literal path") lexical_path = reject_forbidden_path(path, "TMPDIR") try: status = os.lstat(path) diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index d3c20aed4355..5ffd4df1ee20 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -406,14 +406,16 @@ def validate_host_attestation(host: Any) -> dict[str, Any]: "process_group_status", "threshold_reason", } -WATCHDOG_ERROR_CLASSIFICATIONS = { +WATCHDOG_REQUIRED_ERROR_CLASSIFICATIONS = { "configuration_error", "internal_error", "launch_error", "lease_error", + "termination_timeout", +} +WATCHDOG_OPTIONAL_ERROR_CLASSIFICATIONS = { "procfs_error", "signal_error", - "termination_timeout", } @@ -463,7 +465,7 @@ def validate_watchdog_event(event: Any) -> dict[str, Any]: if event["process_group_status"] not in { "not_created", "active", "leader_exited", "signal_error", "termination_timeout", "missing", "sighup_sent", "sigint_sent", - "sigterm_sent", "sigkill_sent"}: + "sigterm_sent", "sigkill_sent", "sigkill_timeout"}: raise TraceError("watchdog JSONL process_group_status is invalid") if not isinstance(event["threshold_reason"], str) or not event["threshold_reason"]: raise TraceError("watchdog JSONL threshold_reason is invalid") @@ -494,10 +496,14 @@ def validate_watchdog_event(event: Any) -> dict[str, Any]: raise TraceError("watchdog JSONL classification is invalid") if type(event["exit_code"]) is not int: raise TraceError("watchdog JSONL exit code is invalid") - requires_error = event["classification"] in WATCHDOG_ERROR_CLASSIFICATIONS - if ("error" in event) != requires_error: + classification = event["classification"] + requires_error = classification in WATCHDOG_REQUIRED_ERROR_CLASSIFICATIONS + allows_error = requires_error or classification in WATCHDOG_OPTIONAL_ERROR_CLASSIFICATIONS + if requires_error and "error" not in event: + raise TraceError("watchdog JSONL error presence does not match classification") + if not allows_error and "error" in event: raise TraceError("watchdog JSONL error presence does not match classification") - if requires_error and (not isinstance(event["error"], str) or not event["error"]): + if "error" in event and (not isinstance(event["error"], str) or not event["error"]): raise TraceError("watchdog JSONL error is invalid") secondary_errors = event.get("secondary_errors") if secondary_errors is not None: From c313eddf221ab35a085b915e88ff1d69facba8df Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 03:32:08 -0700 Subject: [PATCH 27/56] trace : enforce ds4 exporter approval Assisted-by: GPT-5.6 Sol Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 96 +++++++++++++++++++++++- tools/deepseek-v41-trace/README.md | 4 +- tools/deepseek-v41-trace/run_ds4.py | 6 +- tools/deepseek-v41-trace/trace_format.py | 6 ++ 4 files changed, 104 insertions(+), 8 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 9abd27845222..62e458c468f8 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -29,6 +29,9 @@ import verify_ds4_anchors trace.APPROVED_WATCHDOGS[trace.WATCHDOG_SCRIPT_SHA256] = trace.WATCHDOG_REVISION +FIXTURE_DS4_EXPORTER_SHA256 = "3" * 64 +trace.APPROVED_EXPORTERS[FIXTURE_DS4_EXPORTER_SHA256] = trace.DS4_REVISION +run_ds4.APPROVED_EXPORTERS[FIXTURE_DS4_EXPORTER_SHA256] = trace.DS4_REVISION WATCHDOG_EVENTS = [ { @@ -195,7 +198,7 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "runner_script": "/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py", "runner_script_sha256": "2" * 64, "exporter_path": "/Users/oracle/bin/ds4-trace", - "exporter_sha256": "3" * 64, + "exporter_sha256": FIXTURE_DS4_EXPORTER_SHA256, "checkout_path": "/Users/oracle/ds4", "checkout_revision": trace.DS4_REVISION, "command_sha256": "4" * 64, @@ -380,7 +383,7 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "compiler": "clang", "target": "arm64-apple-darwin", "path": "/Users/oracle/bin/ds4-trace", - "sha256": "3" * 64, + "sha256": FIXTURE_DS4_EXPORTER_SHA256, } if is_ds4 else { @@ -1689,6 +1692,49 @@ def wait(timeout: float | None = None) -> int: trace.validate_watchdog_event(final_event) canonical_finals[classification] = final_event + def fail_signal(_pid: int, _signal: int) -> str: + raise watchdog.ProcessGroupError("test signal failure") + + signal_stream = io.StringIO() + signal_logger = watchdog.AuditLogger(signal_stream, wall_clock=lambda: now) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._kill_and_finish( + signal_logger, + guardian, + snapshot, + snapshot.used_bytes, + "procfs_error", + watchdog.EXIT_PROCFS_ERROR, + "test signal failure", + fail_signal, + ) + signal_final = trace.strict_json_loads(signal_stream.getvalue().splitlines()[-1]) + self.assertEqual(signal_final["classification"], "signal_error") + self.assertIsInstance(signal_final["error"], str) + trace.validate_watchdog_event(signal_final) + canonical_finals["signal-error-detail"] = signal_final + + secondary_stream = io.StringIO() + secondary_logger = watchdog.AuditLogger(secondary_stream, wall_clock=lambda: now) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._emit_final( + secondary_logger, + "signal_error", + watchdog.EXIT_SIGNAL_ERROR, + "test secondary error", + snapshot, + snapshot.used_bytes, + guardian, + 0, + "signal_error", + "primary signal failure", + preserve_primary_on_artifact_error=True, + secondary_errors=[{"component": "audit", "detail": "secondary audit failure"}], + ) + secondary_final = trace.strict_json_loads(secondary_stream.getvalue().splitlines()[-1]) + trace.validate_watchdog_event(secondary_final) + canonical_finals["signal-error-secondary"] = secondary_final + class TimeoutProcess(FakeProcess): @staticmethod def wait(timeout: float | None = None) -> int: @@ -2269,6 +2315,30 @@ def test_enforces_watchdog_final_error_classification(self) -> None: replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) trace.TraceBundle(root) + for classification, error in ( + ("internal_error", "internal failure"), + ("signal_error", None)): + with self.subTest( + classification=classification, + secondary=True), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + candidate["secondary_errors"] = [{ + "component": "audit", + "detail": "secondary audit failure", + }] + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + with self.assertRaisesRegex(trace.TraceError, "require a primary signal error"): + trace.TraceBundle(root) + for classification, error in ( ("internal_error", None), ("child_exit", "fabricated error")): @@ -2582,6 +2652,28 @@ def test_rejects_unapproved_ds4_exporter(self) -> None: with self.assertRaisesRegex(preflight.PreflightError, "not approved"): run_ds4.verify_exporter_approval("a" * 64) + def test_bundle_validation_and_comparison_reject_unapproved_ds4_exporter(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + ds4_root = root / "ds4" + llama_root = root / "llama" + with trace.TraceBundleWriter(ds4_root, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(llama_root, manifest("llama.cpp")) as writer: + add_required_events(writer) + approved = dict(trace.APPROVED_EXPORTERS) + try: + trace.APPROVED_EXPORTERS.clear() + with self.assertRaisesRegex(trace.TraceError, "exporter is not approved"): + trace.TraceBundle(ds4_root) + with self.assertRaisesRegex(trace.TraceError, "exporter is not approved"): + trace.command_validate(Namespace(bundle=ds4_root)) + with self.assertRaisesRegex(trace.TraceError, "exporter is not approved"): + trace.command_compare(Namespace(left=ds4_root, right=llama_root, report=None)) + finally: + trace.APPROVED_EXPORTERS.clear() + trace.APPROVED_EXPORTERS.update(approved) + def test_rejects_preflight_audit_mutation(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 3000cf80dcf7..0365f5f1b34d 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -123,11 +123,11 @@ Use `run_llama.py` on the validation host instead of calling the exporter direct ## Apple Metal oracle execution gate -The external ds4 exporter is not present in the pinned canonical checkout. It remains blocked until a separately built executable is reviewed and its exact SHA-256 is added to the otherwise empty `APPROVED_EXPORTERS` map. Approval is checked before the exporter can run, including device-only preflight. `run_ds4.py` requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +The external ds4 exporter is not present in the pinned canonical checkout. It remains blocked until a separately built executable is reviewed and its exact SHA-256 is added to the otherwise empty `APPROVED_EXPORTERS` map. Approval is checked before the exporter can run, including device-only preflight, and is checked again whenever a ds4 bundle is validated or compared. `run_ds4.py` requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. -After the exporter is independently reviewed on an authorized 128 GiB or larger Apple oracle host, add its exact executable SHA-256 and pinned ds4 revision to `APPROVED_EXPORTERS` in `run_ds4.py`; a caller-provided digest alone is not sufficient oracle provenance. The exporter must answer `--dsv41-attest-device Metal0` without loading the model and emit the strict `apple-metal` attestation. Its trace command interface is: +After the exporter is independently reviewed on an authorized 128 GiB or larger Apple oracle host, add its exact executable SHA-256 and pinned ds4 revision to the shared `APPROVED_EXPORTERS` map in `trace_format.py`; a caller-provided digest alone is not sufficient oracle provenance. The exporter must answer `--dsv41-attest-device Metal0` without loading the model and emit the strict `apple-metal` attestation. Its trace command interface is: The unpublished `ds4gguf` documentation revision `e13893ffcb33e90c8852929303e188102df7a8f5` is provenance only. It is not an executable dependency, exporter approval, or fixture source. Executable tests and fixtures stay in this `strix-llama.cpp` stack. diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 2fe59435c73a..f659f859077c 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -22,7 +22,9 @@ ) from trace_format import ( ADMITTED_UBATCH, + APPROVED_EXPORTERS, CORPUS_SHA256, + DS4_REVISION, MODEL_SHA256, NO_EXTERNAL_STATE_STORAGE, TraceBundle, @@ -33,10 +35,6 @@ strict_json_loads, ) -DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" -APPROVED_EXPORTERS: dict[str, str] = {} - - def git_output(checkout: Path, *args: str) -> str: try: return subprocess.check_output( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 5ffd4df1ee20..99caa0fba166 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -15,6 +15,7 @@ TRACE_FORMAT = "dsv41-trace" TRACE_VERSION = 2 DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" +APPROVED_EXPORTERS: dict[str, str] = {} MODEL_SHA256 = "1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42" REPOSITORY = "halo-box/strix-llama.cpp" SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 @@ -507,6 +508,8 @@ def validate_watchdog_event(event: Any) -> dict[str, Any]: raise TraceError("watchdog JSONL error is invalid") secondary_errors = event.get("secondary_errors") if secondary_errors is not None: + if classification != "signal_error" or "error" not in event: + raise TraceError("watchdog JSONL secondary errors require a primary signal error") if not isinstance(secondary_errors, list) or not secondary_errors: raise TraceError("watchdog JSONL secondary errors are invalid") for secondary_error in secondary_errors: @@ -813,6 +816,9 @@ def _validate_manifest(self) -> None: build_sha256 = self.manifest["build"].get("sha256", "") if not isinstance(build_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", build_sha256) is None: raise TraceError("manifest build SHA-256 is invalid") + if self.manifest["runtime"] == "ds4" and ( + APPROVED_EXPORTERS.get(build_sha256) != DS4_REVISION): + raise TraceError("ds4 exporter is not approved for the pinned ds4 revision") for key in build_keys - {"sha256", "number"}: value = self.manifest["build"].get(key) if not isinstance(value, str) or not value: From c5fb91fc78dfd52895c19b41ba078c081127b6c3 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 03:39:24 -0700 Subject: [PATCH 28/56] trace : reject null secondary watchdog errors Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test-deepseek41-trace.py | 92 +++++++++++++++--------- tools/deepseek-v41-trace/trace_format.py | 4 +- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 62e458c468f8..d48a1bc57ec6 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2288,42 +2288,61 @@ def test_enforces_watchdog_final_error_classification(self) -> None: "exit_code": 1, "error": "test internal error", }) - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) / "trace" - with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: - add_required_events(writer) - for phase in ("pre", "post"): - replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, terminal]) - trace.TraceBundle(root) + + def assert_watchdog_final_rejected(candidate: dict[str, object], message: str) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + with self.assertRaisesRegex(trace.TraceError, message): + trace.command_validate(Namespace(bundle=root)) + + def assert_watchdog_final_valid(candidate: dict[str, object]) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + trace.TraceBundle(root) + with mock.patch("sys.stdout", new_callable=io.StringIO): + self.assertEqual(trace.command_validate(Namespace(bundle=root)), 0) + + assert_watchdog_final_valid(terminal) for classification, error in ( ("procfs_error", None), ("procfs_error", "initial snapshot failed"), ("signal_error", None), ("signal_error", "cannot signal process group")): - with self.subTest(classification=classification, error=error), tempfile.TemporaryDirectory() as temp: - root = Path(temp) / "trace" - with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: - add_required_events(writer) + with self.subTest(classification=classification, error=error): candidate = copy.deepcopy(terminal) candidate["classification"] = classification if error is None: candidate.pop("error") else: candidate["error"] = error - for phase in ("pre", "post"): - replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) - trace.TraceBundle(root) + assert_watchdog_final_valid(candidate) + + candidate = copy.deepcopy(terminal) + candidate.update({ + "classification": "signal_error", + "error": "primary signal failure", + "secondary_errors": [{ + "component": "audit", + "detail": "secondary audit failure", + }], + }) + assert_watchdog_final_valid(candidate) for classification, error in ( ("internal_error", "internal failure"), ("signal_error", None)): - with self.subTest( - classification=classification, - secondary=True), tempfile.TemporaryDirectory() as temp: - root = Path(temp) / "trace" - with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: - add_required_events(writer) + with self.subTest(classification=classification, secondary=True): candidate = copy.deepcopy(terminal) candidate["classification"] = classification candidate["secondary_errors"] = [{ @@ -2334,28 +2353,37 @@ def test_enforces_watchdog_final_error_classification(self) -> None: candidate.pop("error") else: candidate["error"] = error - for phase in ("pre", "post"): - replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) - with self.assertRaisesRegex(trace.TraceError, "require a primary signal error"): - trace.TraceBundle(root) + assert_watchdog_final_rejected(candidate, "require a primary signal error") + + for classification, error, secondary_errors, message in ( + ("child_exit", None, None, "require a primary signal error"), + ("internal_error", "internal failure", None, "require a primary signal error"), + ("signal_error", None, None, "require a primary signal error"), + ("signal_error", "primary signal failure", None, "secondary errors are invalid"), + ("signal_error", "primary signal failure", [], "secondary errors are invalid")): + with self.subTest( + classification=classification, + secondary_errors=secondary_errors): + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + candidate["secondary_errors"] = secondary_errors + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + assert_watchdog_final_rejected(candidate, message) for classification, error in ( ("internal_error", None), ("child_exit", "fabricated error")): - with self.subTest(classification=classification), tempfile.TemporaryDirectory() as temp: - root = Path(temp) / "trace" - with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: - add_required_events(writer) + with self.subTest(classification=classification): candidate = copy.deepcopy(terminal) candidate["classification"] = classification if error is None: candidate.pop("error") else: candidate["error"] = error - for phase in ("pre", "post"): - replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) - with self.assertRaisesRegex(trace.TraceError, "error presence does not match"): - trace.TraceBundle(root) + assert_watchdog_final_rejected(candidate, "error presence does not match") def test_rejects_boolean_accelerator_identities(self) -> None: for runtime, field in ( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 99caa0fba166..bdc3eaf33661 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -506,8 +506,8 @@ def validate_watchdog_event(event: Any) -> dict[str, Any]: raise TraceError("watchdog JSONL error presence does not match classification") if "error" in event and (not isinstance(event["error"], str) or not event["error"]): raise TraceError("watchdog JSONL error is invalid") - secondary_errors = event.get("secondary_errors") - if secondary_errors is not None: + if "secondary_errors" in event: + secondary_errors = event["secondary_errors"] if classification != "signal_error" or "error" not in event: raise TraceError("watchdog JSONL secondary errors require a primary signal error") if not isinstance(secondary_errors, list) or not secondary_errors: From a414f2c48f9d701858dd1e87aa6e88670e16944d Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 03:43:35 -0700 Subject: [PATCH 29/56] trace : record Copilot session provenance Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 From 5c029ead8f744ef58ebf6081459127ec96cfaa80 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 03:58:36 -0700 Subject: [PATCH 30/56] trace : bind loaded exporter runtime Bind the full candidate revision, canonical executable, loaded runtime libraries, and the production native manifest writer into the audited trace contract. Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 139 +++++++++- tools/deepseek-v41-trace/CMakeLists.txt | 16 +- tools/deepseek-v41-trace/README.md | 8 +- tools/deepseek-v41-trace/llama-trace.cpp | 319 +++++++++++++++++++---- tools/deepseek-v41-trace/run_ds4.py | 6 + tools/deepseek-v41-trace/run_llama.py | 80 +++++- tools/deepseek-v41-trace/trace_format.py | 69 ++++- 7 files changed, 559 insertions(+), 78 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index d48a1bc57ec6..545dafa122b6 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -5,6 +5,7 @@ import io import json import os +import shutil import subprocess import struct import sys @@ -393,6 +394,19 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "target": "arm64-apple-darwin", "path": "/home/repo/build/bin/llama-deepseek-v41-trace", "sha256": "3" * 64, + "runtime_libraries": [ + { + "role": role, + "path": f"/home/repo/build/bin/{name}", + "sha256": digest * 64, + } + for role, name, digest in ( + ("build-info", "libllama-common.so", "4"), + ("llama", "libllama.so", "5"), + ("ggml", "libggml.so", "6"), + ("selected-backend", "libggml-hip.so", "7"), + ) + ], } ), "model": { @@ -501,7 +515,10 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "revision": "a" * 40, "base_revision": "b" * 40, "diff_sha256": "c" * 64, + "executable_path": result["build"]["path"], "executable_sha256": "3" * 64, + "runtime_libraries_sha256": trace.sha256_bytes( + trace.canonical_json(result["build"]["runtime_libraries"]).encode("ascii")), } else: result["host"] = dict(DS4_HOST_ATTESTATION) @@ -1595,18 +1612,19 @@ def test_approved_watchdog_is_exact(self) -> None: def test_canonical_watchdog_artifacts_embed_and_validate(self) -> None: revision = preflight.WATCHDOG_REVISION repository = Path(__file__).parents[1] - source = subprocess.check_output( - ["git", "show", f"{revision}:scripts/strix_memory_watchdog.py"], + script = repository / "scripts" / "strix_memory_watchdog.py" + self.assertTrue(script.is_file()) + source = script.read_bytes() + self.assertEqual(preflight.sha256_bytes(source), preflight.WATCHDOG_SCRIPT_SHA256) + subprocess.run( + ["git", "merge-base", "--is-ancestor", revision, "HEAD"], cwd=repository, + check=True, ) - self.assertEqual(preflight.sha256_bytes(source), preflight.WATCHDOG_SCRIPT_SHA256) with tempfile.TemporaryDirectory() as temp: root = Path(temp).resolve() - repo = root / "repo" - script = repo / "scripts" / "strix_memory_watchdog.py" - script.parent.mkdir(parents=True) - script.write_bytes(source) + repo = repository watchdog = preflight._load_watchdog_module(script) lease_path = root / "watchdog.lease" @@ -1941,6 +1959,43 @@ def test_rejects_unpinned_ds4_revision(self) -> None: with self.assertRaisesRegex(trace.TraceError, "ds4 revision"): trace.TraceBundle(root) + def test_rejects_unbound_runtime_build_identity(self) -> None: + cases = [] + + ds4_manifest = manifest("ds4") + ds4_manifest["build"]["path"] = "/Users/attacker/unrelated-exporter" + cases.append((ds4_manifest, "ds4 build path")) + + short_revision_manifest = manifest() + short_revision_manifest["revision"] = "a" * 9 + short_revision_manifest["candidate"]["revision"] = "a" * 9 + cases.append((short_revision_manifest, "exact full Git revision")) + + revision_manifest = manifest() + revision_manifest["candidate"]["revision"] = "d" * 40 + cases.append((revision_manifest, "candidate revision")) + + executable_manifest = manifest() + executable_manifest["candidate"]["executable_path"] = "/home/repo/build/bin/other-exporter" + cases.append((executable_manifest, "candidate executable path")) + + library_manifest = manifest() + library_manifest["build"]["runtime_libraries"][0]["sha256"] = "e" * 64 + cases.append((library_manifest, "candidate runtime library identities")) + + library_path_manifest = manifest() + library_path_manifest["build"]["runtime_libraries"][0]["path"] = ( + "/home/repo/build/bin/../substituted/libllama-common.so") + cases.append((library_path_manifest, "runtime library path is not canonical")) + + for trace_manifest, message in cases: + with self.subTest(message=message), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + def test_rejects_unattested_accelerator_identity(self) -> None: for key, value, message in ( ("architecture", "gfx1100", "architecture mismatch"), @@ -2128,7 +2183,7 @@ def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: with self.assertRaisesRegex(trace.TraceError, "environment is not macOS"): trace.TraceBundle(root) - def test_native_exporter_manifest_field_types_validate(self) -> None: + def test_native_complete_manifest_writer_validates(self) -> None: binary = Path(os.environ.get( "DSV41_NATIVE_TRACE_BINARY", Path(__file__).parents[1] / "build-harness" / "bin" / "llama-deepseek-v41-trace", @@ -2140,15 +2195,81 @@ def test_native_exporter_manifest_field_types_validate(self) -> None: with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: add_required_events(writer) manifest_path = root / trace.MANIFEST_NAME - command = [str(binary), "--dsv41-manifest-type-probe", str(manifest_path)] + fixture = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + writer_input = { + key: fixture[key] + for key in ("model", "prompt", "accelerator", "paths", "config", "audits", "expected", "event_count") + } + writer_input["system_info"] = "Linux model-free manifest writer test" + input_path = Path(temp) / "manifest-input.json" + input_path.write_text( + json.dumps(writer_input, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + manifest_path.unlink() + command = [ + str(binary.resolve()), + "--dsv41-manifest-writer-probe", + str(input_path), + str(manifest_path), + ] subprocess.run(command, check=True) native = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=Path(__file__).parents[1], + text=True, + ).strip() + version = subprocess.run( + [str(binary.resolve()), "--version"], + check=True, + capture_output=True, + text=True, + ) + self.assertIn(f"commit {revision}", version.stdout) + self.assertEqual(native["revision"], revision) + self.assertEqual(native["build"]["path"], str(binary.resolve())) + self.assertEqual( + {library["role"] for library in native["build"]["runtime_libraries"]}, + {"build-info", "llama", "ggml", "selected-backend"}, + ) self.assertIsInstance(native["environment"]["command"], str) self.assertEqual(json.loads(native["environment"]["command"]), command) self.assertIs(native["config"]["flash_attention"], True) self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) + attestation = fixture["candidate"] + attestation["revision"] = revision + attestation["executable_path"] = str(binary.resolve()) + attestation["executable_sha256"] = trace.sha256_file(binary) + run_llama.bind_candidate_attestation( + root, + attestation, + native["accelerator"], + binary, + trace.sha256_file(binary), + ) trace.TraceBundle(root) + library = next( + item for item in native["build"]["runtime_libraries"] + if item["role"] == "build-info") + substituted = Path(temp) / "substituted" + substituted.mkdir() + substituted_library = substituted / Path(library["path"]).name + shutil.copy2(library["path"], substituted_library) + rejected = subprocess.run( + [ + str(binary.resolve()), + "--dsv41-runtime-module-path-probe", + str(substituted_library), + ], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("outside the exporter runtime directory", rejected.stderr) + def test_prompt_builder_result_becomes_strict_provenance(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index ff447cde482b..d9726f8b4af1 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -1,8 +1,22 @@ set(TARGET llama-deepseek-v41-trace) +find_package(Git REQUIRED) +execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE DSV41_BUILD_REVISION + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE DSV41_BUILD_REVISION_RESULT) +string(LENGTH "${DSV41_BUILD_REVISION}" DSV41_BUILD_REVISION_LENGTH) +if(NOT DSV41_BUILD_REVISION_RESULT EQUAL 0 OR + NOT DSV41_BUILD_REVISION_LENGTH EQUAL 40 OR + NOT DSV41_BUILD_REVISION MATCHES "^[0-9a-f]+$") + message(FATAL_ERROR "DeepSeek V4.1 trace exporter requires the exact Git revision") +endif() add_executable(${TARGET} llama-trace.cpp) -target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src) target_compile_features(${TARGET} PRIVATE cxx_std_17) +target_compile_definitions(${TARGET} PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) add_executable(${PROMPT_TARGET} prompt-builder.cpp) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 0365f5f1b34d..d41bafa403e8 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -4,7 +4,7 @@ This directory defines version 2 of the cross-runtime trace format used by issue Each trace is a directory: -- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, inference configuration, environment, runtime-specific host evidence, exact execution paths, and content-addressed audit references. +- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, canonical executable path, loaded runtime-library paths and hashes, inference configuration, environment, runtime-specific host evidence, exact execution paths, and content-addressed audit references. - `events.jsonl` is an ordered stream of content-addressed event records. - `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. - `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. @@ -81,7 +81,7 @@ Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the comp `run_llama.py` refuses model execution when swap is enabled, the canonical watchdog lease, heartbeat, or JSONL audit is missing or stale, another unrelated matching model workload is active, or any model/prompt/trace path fails the storage gate. -The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd0747962b`, with `scripts/strix_memory_watchdog.py` SHA-256 `d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f`. Both Python validators and the native exporter reject every other revision or script hash. +The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd0747962b`, with `scripts/strix_memory_watchdog.py` SHA-256 `d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f`. That revision is an ancestor of this stack, so production validates the script in the exact candidate tree instead of copying it from another revision. Both Python validators and the native exporter reject every other revision or script hash. The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. @@ -113,11 +113,11 @@ test "$(awk 'NR > 1 { count++ } END { print count + 0 }' /proc/swaps)" = 0 The expected model digest is `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`, both selected block devices report `ROTA=0`, and `/proc/swaps` has zero entries. The observed planning snapshot had 76366495744 bytes free on `/mnt/models` and 679635001344 bytes free under `/home`; recheck before every run. Keep the unchanged 365713686528-byte GGUF in place on `/mnt/models`. Do not copy the model or place builds, logs, traces, audit files, or temporary files there. Put all of those under `/home`, and never use `/mnt/bigspace`. -The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and rejects a bundle unless the exporter reports the pinned revision and its build SHA-256 matches the executed file. +The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and canonical executed path, and rejects a bundle unless the exporter reports the pinned revision and its build path and SHA-256 match the executed file. The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. The content-addressed memory audit binds the preflight accelerator and storage attestations; the manifest binds the independently repeated native accelerator attestation. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. It rejects tracked or untracked checkout changes and rejects an exporter whose embedded build revision, executable hash, accelerator identity, or loaded model device does not match that attestation. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. The native exporter embeds the full 40-character candidate revision independently of dynamically loaded build-info, resolves its actual executable path, and hashes the loaded build-info, llama, ggml, and selected-backend modules. The launcher rejects tracked or untracked checkout changes, prefix-only revision matches, substituted executable or runtime-library paths, changed runtime-library bytes, and any accelerator or loaded-model device mismatch. `run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, and captures the llama.cpp side. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index eb74d980a4ed..d03117fc26ca 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -33,6 +33,14 @@ extern "C" { #include #include +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#if defined(__APPLE__) +#include +#endif #if defined(__linux__) #include #include @@ -40,6 +48,7 @@ extern "C" { #include #include #endif +#endif #if !defined(_WIN32) #include @@ -49,6 +58,7 @@ namespace fs = std::filesystem; using json = nlohmann::ordered_json; static constexpr int TRACE_VERSION = 2; +static constexpr const char * BUILD_REVISION = DSV41_BUILD_REVISION; #if defined(__linux__) static constexpr const char * WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f"; @@ -111,6 +121,138 @@ static std::vector read_file(const fs::path & path) { return result; } +static fs::path canonical_path(const fs::path & path, const char * label) { + try { + return fs::canonical(path); + } catch (const fs::filesystem_error & error) { + throw std::runtime_error(std::string("cannot resolve ") + label + ": " + error.what()); + } +} + +static fs::path current_executable_path() { +#if defined(_WIN32) + std::vector buffer(32768); + const DWORD size = GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (size == 0 || size >= buffer.size()) { + throw std::runtime_error("cannot query current executable path"); + } + return canonical_path(fs::path(std::wstring(buffer.data(), size)), "current executable"); +#elif defined(__APPLE__) + uint32_t size = 0; + if (_NSGetExecutablePath(nullptr, &size) != -1 || size == 0) { + throw std::runtime_error("cannot query current executable path size"); + } + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) { + throw std::runtime_error("cannot query current executable path"); + } + return canonical_path(buffer.data(), "current executable"); +#elif defined(__linux__) + return canonical_path("/proc/self/exe", "current executable"); +#else +#error unsupported platform +#endif +} + +static fs::path module_path(const void * address) { +#if defined(_WIN32) + HMODULE module = nullptr; + if (!GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(address), + &module)) { + throw std::runtime_error("cannot identify loaded runtime module"); + } + std::vector buffer(32768); + const DWORD size = GetModuleFileNameW(module, buffer.data(), static_cast(buffer.size())); + if (size == 0 || size >= buffer.size()) { + throw std::runtime_error("cannot query loaded runtime module path"); + } + return canonical_path(fs::path(std::wstring(buffer.data(), size)), "loaded runtime module"); +#else + Dl_info info = {}; + if (dladdr(address, &info) == 0 || info.dli_fname == nullptr || info.dli_fname[0] == '\0') { + throw std::runtime_error("cannot identify loaded runtime module"); + } + return canonical_path(info.dli_fname, "loaded runtime module"); +#endif +} + +template +static const void * function_address(T function) { + return reinterpret_cast(reinterpret_cast(function)); +} + +static bool path_is_within(const fs::path & path, const fs::path & root) { + const fs::path relative = path.lexically_relative(root); + return !relative.empty() && *relative.begin() != ".."; +} + +static void require_runtime_module_location(const fs::path & executable, const fs::path & module) { + const fs::path binary_directory = executable.parent_path(); + const fs::path library_directory = binary_directory.parent_path() / "lib"; + if (module != executable && module.parent_path() != binary_directory && + !path_is_within(module, library_directory)) { + throw std::runtime_error("loaded runtime module is outside the exporter runtime directory: " + module.string()); + } +} + +static json runtime_module_json( + const std::string & role, + const fs::path & executable, + const void * address) { + const fs::path path = module_path(address); + require_runtime_module_location(executable, path); + return { + {"role", role}, + {"path", path.string()}, + {"sha256", sha256_file(path)}, + }; +} + +static json runtime_build_json( + const fs::path & executable, + ggml_backend_dev_t selected_device, + char ** argv) { + const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked exporter"); + if (invoked_path != executable) { + throw std::runtime_error("invoked exporter path does not match the running executable"); + } + const std::string revision = BUILD_REVISION; + if (revision.size() != 40 || !std::all_of(revision.begin(), revision.end(), [](unsigned char value) { + return std::isdigit(value) || (value >= 'a' && value <= 'f'); + })) { + throw std::runtime_error("embedded exporter revision is invalid"); + } + const std::string linked_revision = llama_commit(); + const std::string ggml_revision = ggml_commit(); + if (linked_revision.empty() || revision.compare(0, linked_revision.size(), linked_revision) != 0 || + ggml_revision.empty() || revision.compare(0, ggml_revision.size(), ggml_revision) != 0) { + throw std::runtime_error("loaded runtime library revision differs from the exporter revision"); + } + if (selected_device == nullptr) { + throw std::runtime_error("cannot bind a null selected backend device"); + } + ggml_backend_reg_t selected_backend = ggml_backend_dev_backend_reg(selected_device); + if (selected_backend == nullptr) { + throw std::runtime_error("selected backend device has no runtime registry"); + } + return { + {"number", llama_build_number()}, + {"info", llama_build_info()}, + {"compiler", llama_compiler()}, + {"target", llama_build_target()}, + {"path", executable.string()}, + {"sha256", sha256_file(executable)}, + {"runtime_libraries", { + runtime_module_json("build-info", executable, function_address(&llama_commit)), + runtime_module_json("llama", executable, function_address(&llama_model_load_from_file)), + runtime_module_json("ggml", executable, function_address(&ggml_init)), + runtime_module_json("selected-backend", executable, selected_backend), + }}, + }; +} + static std::string required_environment(const char * name) { const char * value = std::getenv(name); if (value == nullptr || value[0] == '\0') { @@ -416,6 +558,8 @@ static std::vector tensor_shape(const ggml_tensor * tensor) { return result; } +static void write_manifest_file(const fs::path & path, const json & manifest); + class trace_writer { public: trace_writer(fs::path root, json manifest) : @@ -528,16 +672,7 @@ class trace_writer { } events.close(); manifest["event_count"] = event_count; - const fs::path output = root / "manifest.json"; - const fs::path temp = output.string() + ".tmp"; - { - std::ofstream stream(temp, std::ios::binary | std::ios::trunc); - stream << manifest.dump() << '\n'; - if (!stream) { - throw std::runtime_error("cannot write trace manifest"); - } - } - fs::rename(temp, output); + write_manifest_file(root / "manifest.json", manifest); } private: @@ -700,40 +835,137 @@ static json storage_policy_json() { }; } -static void write_manifest_type_probe(const fs::path & path, int argc, char ** argv) { - const std::vector bytes = read_file(path); - json manifest = json::parse(bytes.begin(), bytes.end()); - if (!manifest.is_object()) { - throw std::runtime_error("manifest type probe input is not a JSON object"); +static json complete_manifest( + json input, + const fs::path & executable, + ggml_backend_dev_t selected_device, + const std::string & system_info, + int argc, + char ** argv) { + static const std::array required = { + "model", "prompt", "accelerator", "paths", "config", "audits", "expected", "event_count", + }; + if (!input.is_object()) { + throw std::runtime_error("manifest writer input is not a JSON object"); } - common_params params; - params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; -#if defined(__linux__) - manifest["environment"]["system_info"] = runtime_system_info(params); -#endif - manifest["environment"]["command"] = command_line_json(argc, argv); - manifest["config"]["flash_attention"] = flash_attention_enabled(params.flash_attn_type); - manifest["storage_policy"] = storage_policy_json(); + for (const char * key : required) { + if (!input.contains(key) && std::string(key) != "event_count") { + throw std::runtime_error(std::string("manifest writer input is missing ") + key); + } + } + for (const auto & item : input.items()) { + if (std::find_if(required.begin(), required.end(), [&](const char * key) { + return item.key() == key; + }) == required.end()) { + throw std::runtime_error("manifest writer input has unexpected field: " + item.key()); + } + } + json manifest = { + {"trace_format", "dsv41-trace"}, + {"trace_version", TRACE_VERSION}, + {"runtime", "llama.cpp"}, + {"revision", BUILD_REVISION}, + {"build", runtime_build_json(executable, selected_device, argv)}, + {"model", std::move(input["model"])}, + {"prompt", std::move(input["prompt"])}, + {"accelerator", std::move(input["accelerator"])}, + {"paths", std::move(input["paths"])}, + {"storage_policy", storage_policy_json()}, + {"config", std::move(input["config"])}, + {"comparison", { + {"tokens", "exact"}, + {"engram_rows", "exact"}, + {"expert_ids", "exact-original-id-space"}, + {"expert_weights", "byte-identical-f32"}, + {"attention_candidates", "exact"}, + {"logits", "byte-identical-f32"}, + }}, + {"environment", { + {"system_info", system_info}, + {"command", command_line_json(argc, argv)}, + }}, + {"audits", std::move(input["audits"])}, + {"expected", std::move(input["expected"])}, + }; + if (input.contains("event_count")) { + manifest["event_count"] = std::move(input["event_count"]); + } + return manifest; +} +static void write_manifest_file(const fs::path & path, const json & manifest) { const fs::path temp = path.string() + ".tmp"; { std::ofstream stream(temp, std::ios::binary | std::ios::trunc); stream << manifest.dump() << '\n'; if (!stream) { - throw std::runtime_error("cannot write manifest type probe"); + throw std::runtime_error("cannot write trace manifest"); } } fs::rename(temp, path); } +static void write_manifest_probe( + const fs::path & input_path, + const fs::path & output_path, + int argc, + char ** argv) { + const std::vector bytes = read_file(input_path); + json input = json::parse(bytes.begin(), bytes.end()); + if (!input.is_object() || !input.contains("system_info") || !input["system_info"].is_string() || + input["system_info"].get().empty()) { + throw std::runtime_error("manifest writer probe system_info is invalid"); + } + const std::string system_info = input["system_info"]; + input.erase("system_info"); + common_params params; + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + input["config"]["flash_attention"] = flash_attention_enabled(params.flash_attn_type); + common_init(); + ggml_backend_load_all(); + ggml_backend_dev_t device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + write_manifest_file( + output_path, + complete_manifest( + std::move(input), + current_executable_path(), + device, + system_info, + argc, + argv)); +} + int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { - if (argc >= 2 && std::string(argv[1]) == "--dsv41-manifest-type-probe") { + if (argc == 2 && std::string(argv[1]) == "--version") { + common_init(); + ggml_backend_load_all(); + const json build = runtime_build_json( + current_executable_path(), + ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU), + argv); + std::cout << "version: deepseek-v41-trace (build " << build["number"] + << ", commit " << BUILD_REVISION << ")\n"; + std::cout << "built with " << build["compiler"].get() + << " for " << build["target"].get() << '\n'; + return 0; + } + if (argc >= 2 && std::string(argv[1]) == "--dsv41-runtime-module-path-probe") { if (argc != 3) { - throw std::runtime_error("--dsv41-manifest-type-probe requires a manifest path"); + throw std::runtime_error("--dsv41-runtime-module-path-probe requires a module path"); + } + require_runtime_module_location( + current_executable_path(), + canonical_path(argv[2], "runtime module probe")); + return 0; + } + if (argc >= 2 && std::string(argv[1]) == "--dsv41-manifest-writer-probe") { + if (argc != 4) { + throw std::runtime_error( + "--dsv41-manifest-writer-probe requires input and output paths"); } - write_manifest_type_probe(argv[2], argc, argv); + write_manifest_probe(argv[2], argv[3], argc, argv); return 0; } if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { @@ -764,6 +996,7 @@ int main(int argc, char ** argv) { if (params.n_predict < 1) { throw std::runtime_error("-n must request at least one deterministic decode step"); } + const fs::path executable_path = current_executable_path(); const dsv41::storage_attestation model_storage = dsv41::require_nvme_path(params.model.path, "model"); @@ -851,17 +1084,7 @@ int main(int argc, char ** argv) { all_layers[layer] = layer; } - json manifest = { - {"runtime", "llama.cpp"}, - {"revision", llama_commit()}, - {"build", { - {"number", llama_build_number()}, - {"info", llama_build_info()}, - {"compiler", llama_compiler()}, - {"target", llama_build_target()}, - {"path", fs::absolute(argv[0]).lexically_normal().string()}, - {"sha256", sha256_file(fs::absolute(argv[0]).lexically_normal())}, - }}, + json manifest_input = { {"model", { {"path", model_path.string()}, {"architecture", "deepseek41"}, @@ -881,7 +1104,6 @@ int main(int argc, char ** argv) { {"repository", audited_storage["repository"].value("resolved_path", "")}, {"temporary_directory", temporary_storage.resolved_path.string()}, }}, - {"storage_policy", storage_policy_json()}, {"config", { {"context", llama_n_ctx(ctx)}, {"batch", params.n_batch}, @@ -915,18 +1137,6 @@ int main(int argc, char ** argv) { {"candidate_propagation_layers", {24, 28, 32, 36}}, }}, }}, - {"comparison", { - {"tokens", "exact"}, - {"engram_rows", "exact"}, - {"expert_ids", "exact-original-id-space"}, - {"expert_weights", "byte-identical-f32"}, - {"attention_candidates", "exact"}, - {"logits", "byte-identical-f32"}, - }}, - {"environment", { - {"system_info", runtime_system_info(params)}, - {"command", command_line_json(argc, argv)}, - }}, {"audits", { {"memory", memory_audit}, {"swap", swap_audit}, @@ -950,6 +1160,13 @@ int main(int argc, char ** argv) { }}, }}, }; + json manifest = complete_manifest( + std::move(manifest_input), + executable_path, + params.devices[0], + runtime_system_info(params), + argc, + argv); trace_writer writer(output_path, std::move(manifest)); llama_set_eval_callback(ctx, trace_callback, &writer); diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index f659f859077c..e4dd78b84555 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -184,6 +184,12 @@ def bind_oracle_attestation( if "paths" in manifest and manifest["paths"] != paths: raise PreflightError("ds4 trace execution paths differ from preflight") manifest["paths"] = paths + build = manifest.get("build") + if not isinstance(build, dict) or build.get("path") != paths["exporter"]: + raise PreflightError("ds4 trace build path differs from the executed exporter") + runner = audit.get("runner") + if not isinstance(runner, dict) or build.get("sha256") != runner.get("exporter_sha256"): + raise PreflightError("ds4 trace build SHA-256 differs from the executed exporter") storage_policy = audit.get("storage_policy") if storage_policy != NO_EXTERNAL_STATE_STORAGE: raise PreflightError("ds4 external cache/state storage policy is invalid") diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 72029a5aa50f..82765d6fb4e7 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -33,6 +33,8 @@ REQUIRED_EXPERT_SLOTS, TraceBundle, TraceError, + canonical_json, + sha256_bytes, sha256_file, strict_json_loads, ) @@ -45,8 +47,12 @@ def git_output(repo: Path, *args: str) -> bytes: raise PreflightError(f"git {' '.join(args)} failed: {error}") from error -def candidate_attestation(args: argparse.Namespace, exporter_sha256: str) -> dict[str, str]: +def candidate_attestation( + args: argparse.Namespace, + exporter: Path, + exporter_sha256: str) -> dict[str, str]: repo = resolved(args.repo) + exporter = resolved(exporter) revision = git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() base_revision = git_output(repo, "rev-parse", args.base_revision).decode("ascii").strip() if revision != args.candidate_revision: @@ -89,14 +95,71 @@ def candidate_attestation(args: argparse.Namespace, exporter_sha256: str) -> dic "revision": revision, "base_revision": base_revision, "diff_sha256": diff_sha256, + "executable_path": str(exporter), "executable_sha256": exporter_sha256, } +def _path_is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + return True + except ValueError: + return False + + +def validate_runtime_build( + manifest: dict[str, object], + *, + exporter: Path, + exporter_sha256: str, + candidate_revision: str) -> str: + build = manifest.get("build") + if not isinstance(build, dict): + raise PreflightError("llama trace build identity is missing") + exporter = resolved(exporter) + if build.get("path") != str(exporter): + raise PreflightError("llama trace build path does not match the executed exporter") + if build.get("sha256") != exporter_sha256: + raise PreflightError("llama trace build SHA-256 does not match the executed exporter") + if manifest.get("revision") != candidate_revision: + raise PreflightError("llama trace build revision does not match the exact candidate revision") + libraries = build.get("runtime_libraries") + if not isinstance(libraries, list): + raise PreflightError("llama trace runtime library identities are missing") + expected_roles = {"build-info", "llama", "ggml", "selected-backend"} + roles = set() + binary_directory = exporter.parent + library_directory = binary_directory.parent / "lib" + for library in libraries: + if not isinstance(library, dict): + raise PreflightError("llama trace runtime library identity is invalid") + role = library.get("role") + path_value = library.get("path") + digest = library.get("sha256") + if role not in expected_roles or role in roles: + raise PreflightError("llama trace runtime library role is invalid") + roles.add(role) + if not isinstance(path_value, str): + raise PreflightError("llama trace runtime library path is invalid") + path = resolved(Path(path_value)) + if path_value != str(path): + raise PreflightError("llama trace runtime library path is not canonical") + if path != exporter and path.parent != binary_directory and not _path_is_within(path, library_directory): + raise PreflightError("llama trace runtime library is outside the exporter runtime directory") + if not path.is_file() or not isinstance(digest, str) or sha256_file(path) != digest: + raise PreflightError("llama trace runtime library SHA-256 mismatch") + if roles != expected_roles: + raise PreflightError("llama trace runtime library identities are incomplete") + return sha256_bytes(canonical_json(libraries).encode("ascii")) + + def bind_candidate_attestation( output: Path, attestation: dict[str, str], - accelerator: dict[str, object]) -> None: + accelerator: dict[str, object], + exporter: Path, + exporter_sha256: str) -> None: manifest_path = safe_trace_path(output, "manifest.json") try: manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) @@ -104,7 +167,14 @@ def bind_candidate_attestation( raise PreflightError(f"cannot bind candidate attestation: {error}") from error if manifest.get("accelerator") != accelerator: raise PreflightError("llama trace accelerator attestation differs from the preflight query") - manifest["candidate"] = attestation + bound_attestation = dict(attestation) + bound_attestation["runtime_libraries_sha256"] = validate_runtime_build( + manifest, + exporter=exporter, + exporter_sha256=exporter_sha256, + candidate_revision=attestation["revision"], + ) + manifest["candidate"] = bound_attestation temp = manifest_path.with_suffix(".tmp") temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") os.replace(temp, manifest_path) @@ -279,7 +349,7 @@ def main() -> int: model_sha256=model_sha256, target_tokens=args.context - args.decode_steps, ) - attestation = candidate_attestation(args, exporter_sha256) + attestation = candidate_attestation(args, exporter, exporter_sha256) output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") @@ -331,7 +401,7 @@ def main() -> int: post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) - bind_candidate_attestation(output, attestation, accelerator) + bind_candidate_attestation(output, attestation, accelerator, exporter, exporter_sha256) bundle = TraceBundle(output) if bundle.manifest.get("runtime") != "llama.cpp": raise PreflightError("llama exporter wrote a non-llama.cpp trace") diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index bdc3eaf33661..8b2b335e96d5 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -9,7 +9,7 @@ import struct import sys from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, BinaryIO, Iterable TRACE_FORMAT = "dsv41-trace" @@ -805,10 +805,13 @@ def _validate_manifest(self) -> None: raise TraceError("manifest revision is invalid") if self.manifest["runtime"] == "ds4" and self.manifest["revision"] != DS4_REVISION: raise TraceError(f"ds4 revision must be {DS4_REVISION}") + if self.manifest["runtime"] == "llama.cpp" and re.fullmatch( + r"[0-9a-f]{40}", self.manifest["revision"]) is None: + raise TraceError("llama.cpp revision must be the exact full Git revision") if not isinstance(self.manifest["build"], dict): raise TraceError("manifest build is invalid") build_keys = ( - {"number", "info", "compiler", "target", "path", "sha256"} + {"number", "info", "compiler", "target", "path", "sha256", "runtime_libraries"} if self.manifest["runtime"] == "llama.cpp" else {"compiler", "target", "path", "sha256"} ) @@ -819,14 +822,38 @@ def _validate_manifest(self) -> None: if self.manifest["runtime"] == "ds4" and ( APPROVED_EXPORTERS.get(build_sha256) != DS4_REVISION): raise TraceError("ds4 exporter is not approved for the pinned ds4 revision") - for key in build_keys - {"sha256", "number"}: + for key in build_keys - {"sha256", "number", "runtime_libraries"}: value = self.manifest["build"].get(key) if not isinstance(value, str) or not value: raise TraceError(f"manifest build {key} is invalid") if "number" in build_keys and type(self.manifest["build"]["number"]) is not int: raise TraceError("manifest build number is invalid") - if not self.manifest["build"]["path"].startswith("/"): - raise TraceError("manifest build path is not absolute") + build_path = self.manifest["build"]["path"] + if not build_path.startswith("/") or ".." in PurePosixPath(build_path).parts or ( + str(PurePosixPath(build_path)) != build_path): + raise TraceError("manifest build path is not canonical") + if self.manifest["runtime"] == "llama.cpp": + libraries = self.manifest["build"]["runtime_libraries"] + if not isinstance(libraries, list) or len(libraries) != 4: + raise TraceError("manifest runtime library identities are invalid") + roles = set() + for library in libraries: + _require_exact_keys(library, {"role", "path", "sha256"}, "manifest runtime library") + role = library.get("role") + path = library.get("path") + digest = library.get("sha256") + if role not in {"build-info", "llama", "ggml", "selected-backend"}: + raise TraceError("manifest runtime library role is invalid") + if role in roles: + raise TraceError("manifest runtime library role is duplicated") + roles.add(role) + if not isinstance(path, str) or not path.startswith("/") or ".." in PurePosixPath(path).parts or ( + str(PurePosixPath(path)) != path): + raise TraceError("manifest runtime library path is not canonical") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError("manifest runtime library SHA-256 is invalid") + if roles != {"build-info", "llama", "ggml", "selected-backend"}: + raise TraceError("manifest runtime library identities are incomplete") for section in ("model", "prompt"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") @@ -904,6 +931,8 @@ def _validate_manifest(self) -> None: raise TraceError("manifest model path is not bound to execution paths") if self.manifest["prompt"].get("path") != paths["prompt"]: raise TraceError("manifest prompt path is not bound to execution paths") + if self.manifest["runtime"] == "ds4" and self.manifest["build"]["path"] != paths["exporter"]: + raise TraceError("ds4 build path is not bound to the executed exporter") corpus_name = self.manifest["prompt"].get("corpus_name") if corpus_name not in CORPUS_SHA256: raise TraceError("prompt corpus is not in the fixed correctness corpus set") @@ -956,20 +985,44 @@ def _validate_manifest(self) -> None: raise TraceError("llama.cpp candidate attestation is missing") _require_exact_keys( candidate, - {"repository", "revision", "base_revision", "diff_sha256", "executable_sha256"}, + { + "repository", + "revision", + "base_revision", + "diff_sha256", + "executable_path", + "executable_sha256", + "runtime_libraries_sha256", + }, "llama.cpp candidate attestation", ) if candidate.get("repository") != REPOSITORY: raise TraceError(f"candidate repository must be {REPOSITORY}") - for key in ("revision", "base_revision", "diff_sha256", "executable_sha256"): + for key in ( + "revision", + "base_revision", + "diff_sha256", + "executable_sha256", + "runtime_libraries_sha256"): value = candidate.get(key, "") if not isinstance(value, str) or re.fullmatch( r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: raise TraceError(f"candidate {key} is invalid") - if not candidate["revision"].startswith(self.manifest["revision"]): + executable_path = candidate.get("executable_path") + if not isinstance(executable_path, str) or not executable_path.startswith("/") or ( + ".." in PurePosixPath(executable_path).parts or + str(PurePosixPath(executable_path)) != executable_path): + raise TraceError("candidate executable path is invalid") + if candidate["revision"] != self.manifest["revision"]: raise TraceError("candidate revision does not match the exporter build revision") + if candidate.get("executable_path") != self.manifest["build"]["path"]: + raise TraceError("candidate executable path does not match the trace build") if candidate["executable_sha256"] != self.manifest["build"]["sha256"]: raise TraceError("candidate executable SHA-256 does not match the trace build") + runtime_libraries_sha256 = sha256_bytes( + canonical_json(self.manifest["build"]["runtime_libraries"]).encode("ascii")) + if candidate["runtime_libraries_sha256"] != runtime_libraries_sha256: + raise TraceError("candidate runtime library identities do not match the trace build") expected_config = { "layer_count": 40, "vocab_size": 129280, From a93a2c348842f79ec8df899b4109f56b150abb4d Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 04:32:50 -0700 Subject: [PATCH 31/56] trace : bind complete runtime library closure Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- cmake/build-info.cmake | 2 +- ggml/CMakeLists.txt | 2 +- tests/test-deepseek41-trace.py | 277 +++++++++++++++--- tools/deepseek-v41-trace/CMakeLists.txt | 27 +- tools/deepseek-v41-trace/README.md | 6 +- tools/deepseek-v41-trace/llama-trace.cpp | 270 +++++++++++++---- tools/deepseek-v41-trace/run_llama.py | 28 +- .../test-injected-library.cpp | 5 + tools/deepseek-v41-trace/trace_format.py | 54 +++- 9 files changed, 562 insertions(+), 109 deletions(-) create mode 100644 tools/deepseek-v41-trace/test-injected-library.cpp diff --git a/cmake/build-info.cmake b/cmake/build-info.cmake index c7005950c561..edf89ebb1458 100644 --- a/cmake/build-info.cmake +++ b/cmake/build-info.cmake @@ -18,7 +18,7 @@ endif() # Get the commit count and hash if(Git_FOUND) execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + COMMAND ${GIT_EXECUTABLE} rev-parse HEAD WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE HEAD OUTPUT_STRIP_TRAILING_WHITESPACE diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index ba9bc83b9b08..c022dc0d5a09 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -13,7 +13,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH) if(GIT_EXE) # Get current git commit hash - execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD + execute_process(COMMAND ${GIT_EXE} rev-parse HEAD WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GGML_BUILD_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 545dafa122b6..6e432b356f12 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -394,11 +394,12 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "target": "arm64-apple-darwin", "path": "/home/repo/build/bin/llama-deepseek-v41-trace", "sha256": "3" * 64, - "runtime_libraries": [ + "runtime_libraries": sorted([ { - "role": role, + "roles": [role], "path": f"/home/repo/build/bin/{name}", "sha256": digest * 64, + "revision": "a" * 40 if role in {"build-info", "ggml"} else None, } for role, name, digest in ( ("build-info", "libllama-common.so", "4"), @@ -406,7 +407,7 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: ("ggml", "libggml.so", "6"), ("selected-backend", "libggml-hip.so", "7"), ) - ], + ], key=lambda library: library["path"]), } ), "model": { @@ -1988,6 +1989,50 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: "/home/repo/build/bin/../substituted/libllama-common.so") cases.append((library_path_manifest, "runtime library path is not canonical")) + external_library_manifest = manifest() + external_library_manifest["build"]["runtime_libraries"][0]["path"] = ( + "/Users/attacker/libggml-injected.dylib") + external_library_manifest["build"]["runtime_libraries"].sort(key=lambda item: item["path"]) + cases.append((external_library_manifest, "outside the exporter runtime directory")) + + omitted_library_manifest = manifest() + omitted_library_manifest["build"]["runtime_libraries"] = [ + library + for library in omitted_library_manifest["build"]["runtime_libraries"] + if "selected-backend" not in library["roles"] + ] + cases.append((omitted_library_manifest, "runtime library identities are incomplete")) + + duplicate_path_manifest = manifest() + duplicate_path_manifest["build"]["runtime_libraries"][1]["path"] = ( + duplicate_path_manifest["build"]["runtime_libraries"][0]["path"]) + cases.append((duplicate_path_manifest, "runtime library path is duplicated")) + + duplicate_role_manifest = manifest() + duplicate_role_manifest["build"]["runtime_libraries"][1]["roles"] = ( + duplicate_role_manifest["build"]["runtime_libraries"][0]["roles"]) + cases.append((duplicate_role_manifest, "runtime library role is duplicated")) + + unsorted_role_manifest = manifest() + unsorted_role_manifest["build"]["runtime_libraries"][0]["roles"] = ["llama", "build-info"] + cases.append((unsorted_role_manifest, "runtime library roles are not sorted")) + + revision_library_manifest = manifest() + revision_library = next( + library + for library in revision_library_manifest["build"]["runtime_libraries"] + if "build-info" in library["roles"]) + revision_library["revision"] = "b" * 40 + cases.append((revision_library_manifest, "runtime library revision is invalid")) + + unexpected_revision_manifest = manifest() + unexpected_revision = next( + library + for library in unexpected_revision_manifest["build"]["runtime_libraries"] + if not set(library["roles"]) & {"build-info", "ggml"}) + unexpected_revision["revision"] = "a" * 40 + cases.append((unexpected_revision_manifest, "runtime library revision is unexpected")) + for trace_manifest, message in cases: with self.subTest(message=message), tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" @@ -2184,12 +2229,16 @@ def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: trace.TraceBundle(root) def test_native_complete_manifest_writer_validates(self) -> None: - binary = Path(os.environ.get( + production_binary = Path(os.environ.get( "DSV41_NATIVE_TRACE_BINARY", Path(__file__).parents[1] / "build-harness" / "bin" / "llama-deepseek-v41-trace", )) - if not binary.is_file(): - self.skipTest("native trace exporter is not built") + manifest_binary = Path(os.environ.get( + "DSV41_NATIVE_MANIFEST_BINARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "test-deepseek41-trace-manifest", + )) + if not production_binary.is_file() or not manifest_binary.is_file(): + self.skipTest("native trace exporter and manifest harness are not built") with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace with space" with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: @@ -2198,9 +2247,8 @@ def test_native_complete_manifest_writer_validates(self) -> None: fixture = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) writer_input = { key: fixture[key] - for key in ("model", "prompt", "accelerator", "paths", "config", "audits", "expected", "event_count") + for key in ("model", "prompt", "audits", "expected", "event_count") } - writer_input["system_info"] = "Linux model-free manifest writer test" input_path = Path(temp) / "manifest-input.json" input_path.write_text( json.dumps(writer_input, sort_keys=True, separators=(",", ":")) + "\n", @@ -2208,8 +2256,8 @@ def test_native_complete_manifest_writer_validates(self) -> None: ) manifest_path.unlink() command = [ - str(binary.resolve()), - "--dsv41-manifest-writer-probe", + str(manifest_binary.resolve()), + "--write-test-manifest", str(input_path), str(manifest_path), ] @@ -2221,51 +2269,207 @@ def test_native_complete_manifest_writer_validates(self) -> None: text=True, ).strip() version = subprocess.run( - [str(binary.resolve()), "--version"], + [str(production_binary.resolve()), "--version"], check=True, capture_output=True, text=True, ) self.assertIn(f"commit {revision}", version.stdout) self.assertEqual(native["revision"], revision) - self.assertEqual(native["build"]["path"], str(binary.resolve())) + self.assertEqual(native["build"]["path"], str(manifest_binary.resolve())) + self.assertIn("test-only manifest harness", native["build"]["info"]) self.assertEqual( - {library["role"] for library in native["build"]["runtime_libraries"]}, + { + role + for library in native["build"]["runtime_libraries"] + for role in library["roles"] + }, {"build-info", "llama", "ggml", "selected-backend"}, ) + self.assertEqual( + [library["path"] for library in native["build"]["runtime_libraries"]], + sorted(library["path"] for library in native["build"]["runtime_libraries"]), + ) + for library in native["build"]["runtime_libraries"]: + expected_revision = ( + revision + if set(library["roles"]) & {"build-info", "ggml"} + else None + ) + self.assertEqual(library["revision"], expected_revision) self.assertIsInstance(native["environment"]["command"], str) self.assertEqual(json.loads(native["environment"]["command"]), command) self.assertIs(native["config"]["flash_attention"], True) self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) attestation = fixture["candidate"] attestation["revision"] = revision - attestation["executable_path"] = str(binary.resolve()) - attestation["executable_sha256"] = trace.sha256_file(binary) - run_llama.bind_candidate_attestation( - root, - attestation, - native["accelerator"], - binary, - trace.sha256_file(binary), + attestation["executable_path"] = str(manifest_binary.resolve()) + attestation["executable_sha256"] = trace.sha256_file(manifest_binary) + with self.assertRaisesRegex(run_llama.PreflightError, "test-only manifest harness"): + run_llama.bind_candidate_attestation( + root, + attestation, + native["accelerator"], + manifest_binary, + trace.sha256_file(manifest_binary), + ) + with self.assertRaisesRegex(trace.TraceError, "candidate executable path"): + trace.TraceBundle(root) + + for protected_field in ( + "accelerator", "build", "candidate", "comparison", "config", + "environment", "paths", "revision", "runtime", "storage_policy"): + protected_input = dict(writer_input) + protected_input[protected_field] = fixture.get(protected_field, {}) + input_path.write_text( + json.dumps(protected_input, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + rejected = subprocess.run(command, check=False, capture_output=True, text=True) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn(f"unexpected field: {protected_field}", rejected.stderr) + + for option in ("--dsv41-manifest-writer-probe", "--dsv41-runtime-module-path-probe"): + rejected = subprocess.run( + [str(production_binary.resolve()), option], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + + rejected = subprocess.run( + [str(production_binary.resolve()), "--dsv41-attest-device", "CPU"], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("selected device is not a GPU backend", rejected.stderr) + + def test_runtime_build_validator_rejects_closure_substitution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + binary_directory = root / "build" / "bin" + library_directory = root / "build" / "lib" + binary_directory.mkdir(parents=True) + library_directory.mkdir() + exporter = binary_directory / "llama-deepseek-v41-trace" + exporter.write_bytes(b"exporter") + records = [] + for name, roles, content in ( + ("libggml-hip.so", ["selected-backend"], b"backend"), + ("libggml.so", ["ggml"], b"ggml"), + ("libllama-common.so", ["build-info"], b"build"), + ("libllama.so", ["llama"], b"llama"), + ("libggml-blas.so", [], b"blas")): + path = library_directory / name + path.write_bytes(content) + records.append({ + "path": str(path.resolve()), + "sha256": trace.sha256_file(path), + "roles": roles, + "revision": "a" * 40 if set(roles) & {"build-info", "ggml"} else None, + }) + records.sort(key=lambda record: record["path"]) + build_manifest = { + "revision": "a" * 40, + "build": { + "path": str(exporter.resolve()), + "sha256": trace.sha256_file(exporter), + "info": "test", + "runtime_libraries": records, + }, + } + digest = run_llama.validate_runtime_build( + build_manifest, + exporter=exporter, + exporter_sha256=trace.sha256_file(exporter), + candidate_revision="a" * 40, + ) + self.assertEqual( + digest, + trace.sha256_bytes(trace.canonical_json(records).encode("ascii")), ) - trace.TraceBundle(root) - library = next( - item for item in native["build"]["runtime_libraries"] - if item["role"] == "build-info") - substituted = Path(temp) / "substituted" - substituted.mkdir() - substituted_library = substituted / Path(library["path"]).name - shutil.copy2(library["path"], substituted_library) + cases = [] + omitted = copy.deepcopy(build_manifest) + omitted["build"]["runtime_libraries"] = [ + library + for library in omitted["build"]["runtime_libraries"] + if "selected-backend" not in library["roles"] + ] + cases.append((omitted, "identities are incomplete")) + + changed_hash = copy.deepcopy(build_manifest) + changed_hash["build"]["runtime_libraries"][0]["sha256"] = "f" * 64 + cases.append((changed_hash, "SHA-256 mismatch")) + + changed_revision = copy.deepcopy(build_manifest) + revision_record = next( + library + for library in changed_revision["build"]["runtime_libraries"] + if "build-info" in library["roles"]) + revision_record["revision"] = "b" * 40 + cases.append((changed_revision, "revision mismatch")) + + duplicate_role = copy.deepcopy(build_manifest) + role_records = duplicate_role["build"]["runtime_libraries"] + next(library for library in role_records if not library["roles"])["roles"] = ["llama"] + cases.append((duplicate_role, "role is invalid")) + + external = root / "external" / "libggml-injected.so" + external.parent.mkdir() + external.write_bytes(b"injected") + external_manifest = copy.deepcopy(build_manifest) + external_manifest["build"]["runtime_libraries"].append({ + "path": str(external.resolve()), + "sha256": trace.sha256_file(external), + "roles": [], + "revision": None, + }) + external_manifest["build"]["runtime_libraries"].sort(key=lambda record: record["path"]) + cases.append((external_manifest, "outside the exporter runtime directory")) + + duplicate_path = copy.deepcopy(build_manifest) + duplicate_path["build"]["runtime_libraries"][1]["path"] = ( + duplicate_path["build"]["runtime_libraries"][0]["path"]) + cases.append((duplicate_path, "duplicated or unsorted")) + + for candidate, message in cases: + with self.subTest(message=message), self.assertRaisesRegex( + run_llama.PreflightError, message): + run_llama.validate_runtime_build( + candidate, + exporter=exporter, + exporter_sha256=trace.sha256_file(exporter), + candidate_revision="a" * 40, + ) + + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "loader injection test") + def test_native_rejects_injected_project_library(self) -> None: + binary = Path(os.environ.get( + "DSV41_NATIVE_TRACE_BINARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "llama-deepseek-v41-trace", + )) + injected = Path(os.environ.get( + "DSV41_NATIVE_INJECT_LIBRARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "libggml-injected.module", + )) + if not binary.is_file() or not injected.is_file(): + self.skipTest("native trace exporter and injected test library are not built") + with tempfile.TemporaryDirectory() as temp: + external = Path(temp) / injected.name + shutil.copy2(injected, external) + environment = dict(os.environ) + variable = "DYLD_INSERT_LIBRARIES" if sys.platform == "darwin" else "LD_PRELOAD" + environment[variable] = str(external) rejected = subprocess.run( - [ - str(binary.resolve()), - "--dsv41-runtime-module-path-probe", - str(substituted_library), - ], + [str(binary.resolve()), "--version"], check=False, capture_output=True, text=True, + env=environment, ) self.assertNotEqual(rejected.returncode, 0) self.assertIn("outside the exporter runtime directory", rejected.stderr) @@ -2923,6 +3127,11 @@ def test_local_base_regression_requires_attested_oracle_revision(self) -> None: base_manifest = manifest("llama.cpp") base_manifest["revision"] = "b" * 40 base_manifest["candidate"]["revision"] = "b" * 40 + for library in base_manifest["build"]["runtime_libraries"]: + if set(library["roles"]) & {"build-info", "ggml"}: + library["revision"] = "b" * 40 + base_manifest["candidate"]["runtime_libraries_sha256"] = trace.sha256_bytes( + trace.canonical_json(base_manifest["build"]["runtime_libraries"]).encode("ascii")) with trace.TraceBundleWriter(base, base_manifest) as writer: add_required_events(writer) with trace.TraceBundleWriter(integrated, manifest("llama.cpp")) as writer: diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index d9726f8b4af1..5fa172318537 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -28,11 +28,28 @@ if(LLAMA_TOOLS_INSTALL) endif() if(LLAMA_BUILD_TESTS) - if(TEST test-deepseek41-trace) - set_property( - TEST test-deepseek41-trace - APPEND PROPERTY ENVIRONMENT "DSV41_NATIVE_TRACE_BINARY=$") - endif() + set(MANIFEST_TEST_TARGET test-deepseek41-trace-manifest) + add_executable(${MANIFEST_TEST_TARGET} llama-trace.cpp) + target_link_libraries( + ${MANIFEST_TEST_TARGET} + PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + target_include_directories(${MANIFEST_TEST_TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_compile_features(${MANIFEST_TEST_TARGET} PRIVATE cxx_std_17) + target_compile_definitions( + ${MANIFEST_TEST_TARGET} + PRIVATE + DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}" + DSV41_MANIFEST_TEST_HARNESS=1) + + set(INJECTED_TEST_TARGET test-deepseek41-trace-injected) + add_library(${INJECTED_TEST_TARGET} SHARED test-injected-library.cpp) + set_target_properties( + ${INJECTED_TEST_TARGET} + PROPERTIES + PREFIX "" + OUTPUT_NAME "libggml-injected" + SUFFIX ".module") + set(HOST_TEST_TARGET test-deepseek41-trace-host) add_executable(${HOST_TEST_TARGET} test-host-attestation.cpp) target_link_libraries(${HOST_TEST_TARGET} PRIVATE ggml) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index d41bafa403e8..ef29e3488131 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -58,6 +58,8 @@ HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ cmake --build build-dsv41-trace-rocm --config Release -j "$(nproc)" --target \ llama-deepseek-v41-trace \ llama-deepseek-v41-prompt-builder \ + test-deepseek41-trace-manifest \ + test-deepseek41-trace-injected \ test-backend-ops \ test-deepseek41-schema \ test-deepseek41-engram \ @@ -117,7 +119,9 @@ The exporter is intentionally external to the canonical ds4 checkout. It must be The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. The content-addressed memory audit binds the preflight accelerator and storage attestations; the manifest binds the independently repeated native accelerator attestation. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. The native exporter embeds the full 40-character candidate revision independently of dynamically loaded build-info, resolves its actual executable path, and hashes the loaded build-info, llama, ggml, and selected-backend modules. The launcher rejects tracked or untracked checkout changes, prefix-only revision matches, substituted executable or runtime-library paths, changed runtime-library bytes, and any accelerator or loaded-model device mismatch. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. The native exporter embeds the full 40-character candidate revision independently of dynamically loaded build-info, resolves its actual executable path, and enumerates every loaded `llama` and `ggml` project library through the platform loader. It canonicalizes, sorts, and hashes the complete closure, records exact revision evidence for revision-bearing modules, and rejects injected or loader-substituted project libraries outside the exporter `bin` and sibling `lib` roots. The launcher reopens and hashes every recorded module and rejects tracked or untracked checkout changes, prefix-only revision matches, omitted or duplicated roles, substituted executable or runtime-library paths, changed runtime-library bytes, test-only writer output, and any accelerator or loaded-model device mismatch. + +Production installs no manifest-writing or runtime-path probe option. With `LLAMA_BUILD_TESTS`, CMake builds the non-installed `test-deepseek41-trace-manifest` harness from the same native writer implementation. Its input accepts only model, prompt, audit, expected-coverage, and event-count data; protected runtime, accelerator, path, configuration, build, and environment fields are fixed internally, and the output carries a test-only build marker that `run_llama.py` refuses to bind as a candidate. `run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, and captures the llama.cpp side. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index d03117fc26ca..3f6ce551ca6c 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -27,6 +27,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ extern "C" { #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN +#include #include #else #include @@ -197,17 +199,120 @@ static void require_runtime_module_location(const fs::path & executable, const f } } -static json runtime_module_json( - const std::string & role, +static bool is_project_runtime_library(const fs::path & path) { + std::string name = path.filename().string(); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { + return static_cast(std::tolower(value)); + }); +#if defined(_WIN32) + return name.rfind("llama", 0) == 0 || name.rfind("ggml", 0) == 0 || + name.rfind("libllama", 0) == 0 || name.rfind("libggml", 0) == 0; +#else + return name.rfind("libllama", 0) == 0 || name.rfind("libggml", 0) == 0 || + name.rfind("ggml", 0) == 0; +#endif +} + +static std::set loaded_project_runtime_libraries() { + std::set result; +#if defined(_WIN32) + const HANDLE snapshot = CreateToolhelp32Snapshot( + TH32CS_SNAPMODULE | TH32CS_SNAPMODULE32, + GetCurrentProcessId()); + if (snapshot == INVALID_HANDLE_VALUE) { + throw std::runtime_error("cannot enumerate loaded runtime modules"); + } + MODULEENTRY32W entry = {}; + entry.dwSize = sizeof(entry); + if (!Module32FirstW(snapshot, &entry)) { + CloseHandle(snapshot); + throw std::runtime_error("cannot read loaded runtime modules"); + } + do { + const fs::path path = canonical_path(entry.szExePath, "loaded runtime module"); + if (is_project_runtime_library(path)) { + result.insert(path); + } + } while (Module32NextW(snapshot, &entry)); + CloseHandle(snapshot); +#elif defined(__APPLE__) + const uint32_t count = _dyld_image_count(); + for (uint32_t index = 0; index < count; ++index) { + const char * name = _dyld_get_image_name(index); + if (name != nullptr && name[0] != '\0') { + const fs::path path = canonical_path(name, "loaded runtime module"); + if (is_project_runtime_library(path)) { + result.insert(path); + } + } + } +#elif defined(__linux__) + std::ifstream maps("/proc/self/maps"); + if (!maps) { + throw std::runtime_error("cannot enumerate loaded runtime modules"); + } + std::string line; + while (std::getline(maps, line)) { + const size_t path_start = line.find('/'); + if (path_start == std::string::npos) { + continue; + } + const fs::path path = canonical_path(line.substr(path_start), "loaded runtime module"); + if (is_project_runtime_library(path)) { + result.insert(path); + } + } +#endif + return result; +} + +static json runtime_libraries_json( const fs::path & executable, - const void * address) { - const fs::path path = module_path(address); - require_runtime_module_location(executable, path); - return { - {"role", role}, - {"path", path.string()}, - {"sha256", sha256_file(path)}, - }; + ggml_backend_dev_t selected_device, + const std::string & revision) { + if (selected_device == nullptr) { + throw std::runtime_error("cannot bind a null selected backend device"); + } + ggml_backend_reg_t selected_backend = ggml_backend_dev_backend_reg(selected_device); + if (selected_backend == nullptr) { + throw std::runtime_error("selected backend device has no runtime registry"); + } + const fs::path build_info_module = module_path(function_address(&llama_commit)); + const fs::path llama_module = module_path(function_address(&llama_model_load_from_file)); + const fs::path ggml_module = module_path(function_address(&ggml_init)); + const fs::path selected_backend_module = module_path(selected_backend); + std::set libraries = loaded_project_runtime_libraries(); + libraries.insert(build_info_module); + libraries.insert(llama_module); + libraries.insert(ggml_module); + libraries.insert(selected_backend_module); + + json result = json::array(); + for (const fs::path & library : libraries) { + require_runtime_module_location(executable, library); + std::vector roles; + if (library == build_info_module) { + roles.push_back("build-info"); + } + if (library == llama_module) { + roles.push_back("llama"); + } + if (library == ggml_module) { + roles.push_back("ggml"); + } + if (library == selected_backend_module) { + roles.push_back("selected-backend"); + } + std::sort(roles.begin(), roles.end()); + const bool revision_bearing = library == build_info_module || library == ggml_module; + result.push_back({ + {"path", library.string()}, + {"sha256", sha256_file(library)}, + {"roles", std::move(roles)}, + {"revision", revision_bearing ? json(revision) : json(nullptr)}, + }); + } + return result; } static json runtime_build_json( @@ -226,30 +331,24 @@ static json runtime_build_json( } const std::string linked_revision = llama_commit(); const std::string ggml_revision = ggml_commit(); - if (linked_revision.empty() || revision.compare(0, linked_revision.size(), linked_revision) != 0 || - ggml_revision.empty() || revision.compare(0, ggml_revision.size(), ggml_revision) != 0) { - throw std::runtime_error("loaded runtime library revision differs from the exporter revision"); - } - if (selected_device == nullptr) { - throw std::runtime_error("cannot bind a null selected backend device"); - } - ggml_backend_reg_t selected_backend = ggml_backend_dev_backend_reg(selected_device); - if (selected_backend == nullptr) { - throw std::runtime_error("selected backend device has no runtime registry"); + if (linked_revision != revision || ggml_revision != revision) { + throw std::runtime_error( + "loaded runtime library revision differs from the exporter revision: llama=" + + linked_revision + ", ggml=" + ggml_revision + ", exporter=" + revision); } +#if defined(DSV41_MANIFEST_TEST_HARNESS) + const std::string build_info = std::string(llama_build_info()) + " [test-only manifest harness]"; +#else + const std::string build_info = llama_build_info(); +#endif return { {"number", llama_build_number()}, - {"info", llama_build_info()}, + {"info", build_info}, {"compiler", llama_compiler()}, {"target", llama_build_target()}, {"path", executable.string()}, {"sha256", sha256_file(executable)}, - {"runtime_libraries", { - runtime_module_json("build-info", executable, function_address(&llama_commit)), - runtime_module_json("llama", executable, function_address(&llama_model_load_from_file)), - runtime_module_json("ggml", executable, function_address(&ggml_init)), - runtime_module_json("selected-backend", executable, selected_backend), - }}, + {"runtime_libraries", runtime_libraries_json(executable, selected_device, revision)}, }; } @@ -905,6 +1004,7 @@ static void write_manifest_file(const fs::path & path, const json & manifest) { fs::rename(temp, path); } +#if defined(DSV41_MANIFEST_TEST_HARNESS) static void write_manifest_probe( const fs::path & input_path, const fs::path & output_path, @@ -912,15 +1012,79 @@ static void write_manifest_probe( char ** argv) { const std::vector bytes = read_file(input_path); json input = json::parse(bytes.begin(), bytes.end()); - if (!input.is_object() || !input.contains("system_info") || !input["system_info"].is_string() || - input["system_info"].get().empty()) { - throw std::runtime_error("manifest writer probe system_info is invalid"); - } - const std::string system_info = input["system_info"]; - input.erase("system_info"); - common_params params; - params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; - input["config"]["flash_attention"] = flash_attention_enabled(params.flash_attn_type); + static const std::array allowed = { + "model", "prompt", "audits", "expected", "event_count", + }; + if (!input.is_object()) { + throw std::runtime_error("manifest writer test input is not a JSON object"); + } + for (const char * key : allowed) { + if (!input.contains(key)) { + throw std::runtime_error(std::string("manifest writer test input is missing ") + key); + } + } + for (const auto & item : input.items()) { + if (std::find_if(allowed.begin(), allowed.end(), [&](const char * key) { + return item.key() == key; + }) == allowed.end()) { + throw std::runtime_error("manifest writer test input has unexpected field: " + item.key()); + } + } + input["accelerator"] = { + {"format", "dsv41-accelerator-attestation"}, + {"version", 2}, + {"runtime_kind", "strix-rocm"}, + {"platform", "linux"}, + {"backend", "ROCm"}, + {"backend_device", "ROCm0"}, + {"backend_description", "AMD Radeon Graphics"}, + {"pci_device_id", "0000:c1:00.0"}, + {"kfd_node", 1}, + {"gpu_id", 42}, + {"gfx_target_version", 110501}, + {"architecture", "gfx1151"}, + {"source", "linux-kfd-sysfs"}, + }; + input["paths"] = { + {"model", "/mnt/models/model.gguf"}, + {"prompt", "/home/prompt.txt"}, + {"output", "/home"}, + {"repository", "/home/repo"}, + {"temporary_directory", "/home/tmp"}, + }; + input["config"] = { + {"context", 3}, + {"batch", 2048}, + {"ubatch", 32}, + {"device", "ROCm0"}, + {"device_architecture", "gfx1151"}, + {"device_pci_id", "0000:c1:00.0"}, + {"decode_steps", 1}, + {"kv_type_k", "f16"}, + {"kv_type_v", "f16"}, + {"flash_attention", true}, + {"gpu_layers", 99}, + {"load_mode", 0}, + {"expert_cache_slots", 192}, + {"expert_cache_bytes", UINT64_C(76441190400)}, + {"tokenizer_add_bos", true}, + {"tokenizer_parse_special", true}, + {"deepseek41", { + {"layer_count", 40}, + {"vocab_size", 129280}, + {"engram_layers", {1, 14}}, + {"engram_rows_per_token", 24}, + {"expert_count", 384}, + {"experts_used", 6}, + {"candidate_source_layer", 20}, + {"candidate_topk_blocks", 2048}, + {"candidate_block_size", 8}, + {"index_top_k", 512}, + {"raw_attention_layers", {0, 1}}, + {"raw_attention_width", 128}, + {"candidate_propagation_layers", {24, 28, 32, 36}}, + }}, + }; common_init(); ggml_backend_load_all(); ggml_backend_dev_t device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); @@ -930,11 +1094,25 @@ static void write_manifest_probe( std::move(input), current_executable_path(), device, - system_info, + "Linux model-free manifest writer test", argc, argv)); } +int main(int argc, char ** argv) { + std::setlocale(LC_NUMERIC, "C"); + try { + if (argc != 4 || std::string(argv[1]) != "--write-test-manifest") { + throw std::runtime_error("test manifest writer requires input and output paths"); + } + write_manifest_probe(argv[2], argv[3], argc, argv); + return 0; + } catch (const std::exception & error) { + std::fprintf(stderr, "test-deepseek-v41-manifest: %s\n", error.what()); + return 1; + } +} +#else int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { @@ -951,23 +1129,6 @@ int main(int argc, char ** argv) { << " for " << build["target"].get() << '\n'; return 0; } - if (argc >= 2 && std::string(argv[1]) == "--dsv41-runtime-module-path-probe") { - if (argc != 3) { - throw std::runtime_error("--dsv41-runtime-module-path-probe requires a module path"); - } - require_runtime_module_location( - current_executable_path(), - canonical_path(argv[2], "runtime module probe")); - return 0; - } - if (argc >= 2 && std::string(argv[1]) == "--dsv41-manifest-writer-probe") { - if (argc != 4) { - throw std::runtime_error( - "--dsv41-manifest-writer-probe requires input and output paths"); - } - write_manifest_probe(argv[2], argv[3], argc, argv); - return 0; - } if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { common_init(); ggml_backend_load_all(); @@ -1214,3 +1375,4 @@ int main(int argc, char ** argv) { return 1; } } +#endif diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 82765d6fb4e7..23fb80aececf 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -124,27 +124,45 @@ def validate_runtime_build( raise PreflightError("llama trace build SHA-256 does not match the executed exporter") if manifest.get("revision") != candidate_revision: raise PreflightError("llama trace build revision does not match the exact candidate revision") + if "test-only manifest harness" in str(build.get("info", "")): + raise PreflightError("llama trace was produced by the test-only manifest harness") libraries = build.get("runtime_libraries") - if not isinstance(libraries, list): + if not isinstance(libraries, list) or not libraries: raise PreflightError("llama trace runtime library identities are missing") expected_roles = {"build-info", "llama", "ggml", "selected-backend"} roles = set() + paths = set() + previous_path = None binary_directory = exporter.parent library_directory = binary_directory.parent / "lib" for library in libraries: if not isinstance(library, dict): raise PreflightError("llama trace runtime library identity is invalid") - role = library.get("role") path_value = library.get("path") digest = library.get("sha256") - if role not in expected_roles or role in roles: - raise PreflightError("llama trace runtime library role is invalid") - roles.add(role) + library_roles = library.get("roles") + revision = library.get("revision") + if not isinstance(library_roles, list) or library_roles != sorted(library_roles) or any( + role not in expected_roles for role in library_roles): + raise PreflightError("llama trace runtime library roles are invalid") + for role in library_roles: + if role in roles: + raise PreflightError("llama trace runtime library role is invalid") + roles.add(role) + if bool(set(library_roles) & {"build-info", "ggml"}): + if revision != candidate_revision: + raise PreflightError("llama trace runtime library revision mismatch") + elif revision is not None: + raise PreflightError("llama trace runtime library revision is unexpected") if not isinstance(path_value, str): raise PreflightError("llama trace runtime library path is invalid") path = resolved(Path(path_value)) if path_value != str(path): raise PreflightError("llama trace runtime library path is not canonical") + if path in paths or (previous_path is not None and str(path) <= str(previous_path)): + raise PreflightError("llama trace runtime library paths are duplicated or unsorted") + paths.add(path) + previous_path = path if path != exporter and path.parent != binary_directory and not _path_is_within(path, library_directory): raise PreflightError("llama trace runtime library is outside the exporter runtime directory") if not path.is_file() or not isinstance(digest, str) or sha256_file(path) != digest: diff --git a/tools/deepseek-v41-trace/test-injected-library.cpp b/tools/deepseek-v41-trace/test-injected-library.cpp new file mode 100644 index 000000000000..cd50087a639d --- /dev/null +++ b/tools/deepseek-v41-trace/test-injected-library.cpp @@ -0,0 +1,5 @@ +extern "C" int dsv41_test_injected_library(void); + +extern "C" int dsv41_test_injected_library(void) { + return 1; +} diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 8b2b335e96d5..306a0317dc1a 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -834,25 +834,63 @@ def _validate_manifest(self) -> None: raise TraceError("manifest build path is not canonical") if self.manifest["runtime"] == "llama.cpp": libraries = self.manifest["build"]["runtime_libraries"] - if not isinstance(libraries, list) or len(libraries) != 4: + if not isinstance(libraries, list) or not libraries: raise TraceError("manifest runtime library identities are invalid") roles = set() + paths = set() + previous_path = None + expected_roles = {"build-info", "llama", "ggml", "selected-backend"} + executable_path = PurePosixPath(build_path) + binary_directory = executable_path.parent + library_directory = binary_directory.parent / "lib" for library in libraries: - _require_exact_keys(library, {"role", "path", "sha256"}, "manifest runtime library") - role = library.get("role") + _require_exact_keys( + library, + {"path", "sha256", "roles", "revision"}, + "manifest runtime library", + ) path = library.get("path") digest = library.get("sha256") - if role not in {"build-info", "llama", "ggml", "selected-backend"}: - raise TraceError("manifest runtime library role is invalid") - if role in roles: + library_roles = library.get("roles") + revision = library.get("revision") + if not isinstance(library_roles, list) or any( + not isinstance(role, str) or role not in expected_roles + for role in library_roles): + raise TraceError("manifest runtime library roles are invalid") + if len(library_roles) != len(set(library_roles)): raise TraceError("manifest runtime library role is duplicated") - roles.add(role) + if library_roles != sorted(library_roles): + raise TraceError("manifest runtime library roles are not sorted") + for role in library_roles: + if role in roles: + raise TraceError("manifest runtime library role is duplicated") + roles.add(role) if not isinstance(path, str) or not path.startswith("/") or ".." in PurePosixPath(path).parts or ( str(PurePosixPath(path)) != path): raise TraceError("manifest runtime library path is not canonical") + if path in paths: + raise TraceError("manifest runtime library path is duplicated") + if previous_path is not None and path <= previous_path: + raise TraceError("manifest runtime library paths are not sorted") + runtime_path = PurePosixPath(path) + try: + runtime_path.relative_to(library_directory) + in_library_directory = True + except ValueError: + in_library_directory = False + if runtime_path != executable_path and runtime_path.parent != binary_directory and ( + not in_library_directory): + raise TraceError("manifest runtime library is outside the exporter runtime directory") + paths.add(path) + previous_path = path if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError("manifest runtime library SHA-256 is invalid") - if roles != {"build-info", "llama", "ggml", "selected-backend"}: + if set(library_roles) & {"build-info", "ggml"}: + if revision != self.manifest["revision"]: + raise TraceError("manifest runtime library revision is invalid") + elif revision is not None: + raise TraceError("manifest runtime library revision is unexpected") + if roles != expected_roles: raise TraceError("manifest runtime library identities are incomplete") for section in ("model", "prompt"): if not isinstance(self.manifest[section], dict): From 9996323dcf59fc73f18374e2e7fbb81e0b7e84f9 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 04:34:01 -0700 Subject: [PATCH 32/56] trace : ignore non-project loader images Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tools/deepseek-v41-trace/llama-trace.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 3f6ce551ca6c..5594c1b5e840 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -229,9 +229,9 @@ static std::set loaded_project_runtime_libraries() { throw std::runtime_error("cannot read loaded runtime modules"); } do { - const fs::path path = canonical_path(entry.szExePath, "loaded runtime module"); - if (is_project_runtime_library(path)) { - result.insert(path); + const fs::path reported_path = entry.szExePath; + if (is_project_runtime_library(reported_path)) { + result.insert(canonical_path(reported_path, "loaded runtime module")); } } while (Module32NextW(snapshot, &entry)); CloseHandle(snapshot); @@ -240,9 +240,9 @@ static std::set loaded_project_runtime_libraries() { for (uint32_t index = 0; index < count; ++index) { const char * name = _dyld_get_image_name(index); if (name != nullptr && name[0] != '\0') { - const fs::path path = canonical_path(name, "loaded runtime module"); - if (is_project_runtime_library(path)) { - result.insert(path); + const fs::path reported_path = name; + if (is_project_runtime_library(reported_path)) { + result.insert(canonical_path(reported_path, "loaded runtime module")); } } } @@ -257,9 +257,9 @@ static std::set loaded_project_runtime_libraries() { if (path_start == std::string::npos) { continue; } - const fs::path path = canonical_path(line.substr(path_start), "loaded runtime module"); - if (is_project_runtime_library(path)) { - result.insert(path); + const fs::path reported_path = line.substr(path_start); + if (is_project_runtime_library(reported_path)) { + result.insert(canonical_path(reported_path, "loaded runtime module")); } } #endif From a246459d543631d99b78a7a865ab93f04bf76324 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 04:35:15 -0700 Subject: [PATCH 33/56] tests : verify unsigned manifest rejection Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 6e432b356f12..60af05b69acd 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2313,7 +2313,7 @@ def test_native_complete_manifest_writer_validates(self) -> None: manifest_binary, trace.sha256_file(manifest_binary), ) - with self.assertRaisesRegex(trace.TraceError, "candidate executable path"): + with self.assertRaisesRegex(trace.TraceError, "missing candidate"): trace.TraceBundle(root) for protected_field in ( @@ -2345,7 +2345,7 @@ def test_native_complete_manifest_writer_validates(self) -> None: text=True, ) self.assertNotEqual(rejected.returncode, 0) - self.assertIn("selected device is not a GPU backend", rejected.stderr) + self.assertIn("selected execution device must be ROCm0", rejected.stderr) def test_runtime_build_validator_rejects_closure_substitution(self) -> None: with tempfile.TemporaryDirectory() as temp: From 88c27b4d708443da1c5a94290744f7ab10aa52e7 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 06:30:03 -0700 Subject: [PATCH 34/56] deepseek41 : seal correctness trace bundles Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/CMakeLists.txt | 8 + tests/test-deepseek41-trace.py | 975 ++++++++++++-- tools/deepseek-v41-trace/CMakeLists.txt | 155 ++- tools/deepseek-v41-trace/README.md | 58 +- .../generate-runtime-receipt.py | 134 ++ tools/deepseek-v41-trace/llama-trace.cpp | 609 +++++++-- tools/deepseek-v41-trace/run_ds4.py | 50 +- tools/deepseek-v41-trace/run_llama.py | 159 ++- tools/deepseek-v41-trace/run_matrix.py | 46 +- tools/deepseek-v41-trace/test-install.cmake | 64 + tools/deepseek-v41-trace/trace_format.py | 1141 ++++++++++++++++- .../verify-runtime-install.py | 88 ++ 12 files changed, 3154 insertions(+), 333 deletions(-) create mode 100644 tools/deepseek-v41-trace/generate-runtime-receipt.py create mode 100644 tools/deepseek-v41-trace/test-install.cmake create mode 100644 tools/deepseek-v41-trace/verify-runtime-install.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d4cc702904b5..135c5925092e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -214,6 +214,14 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test-deepseek41-trace.py ) + if(BUILD_SHARED_LIBS AND LLAMA_BUILD_TOOLS AND NOT EMSCRIPTEN) + set_property( + TEST test-deepseek41-trace + APPEND PROPERTY ENVIRONMENT + "DSV41_NATIVE_TRACE_BINARY=$" + "DSV41_NATIVE_MANIFEST_BINARY=$" + "DSV41_NATIVE_INJECT_LIBRARY=$") + endif() endif() set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 60af05b69acd..3164520fca94 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -10,6 +10,7 @@ import struct import sys import tempfile +import time import unittest from argparse import Namespace from datetime import datetime, timezone @@ -33,6 +34,13 @@ FIXTURE_DS4_EXPORTER_SHA256 = "3" * 64 trace.APPROVED_EXPORTERS[FIXTURE_DS4_EXPORTER_SHA256] = trace.DS4_REVISION run_ds4.APPROVED_EXPORTERS[FIXTURE_DS4_EXPORTER_SHA256] = trace.DS4_REVISION +TEST_AUTH_ISSUED = int(time.time()) - 60 +TEST_AUTH_EXPIRES = TEST_AUTH_ISSUED + 3600 +TEST_CHALLENGE = "d" * 64 +TEST_RUN_IDS = { + "llama.cpp": "strix-llama-test-run", + "ds4": "apple-ds4-test-run", +} WATCHDOG_EVENTS = [ { @@ -376,9 +384,49 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: is_ds4 = runtime == "ds4" storage = DS4_STORAGE_ATTESTATION if is_ds4 else STORAGE_ATTESTATION audit_kinds = ("memory", "swap", "runner") if is_ds4 else ("memory", "swap", "watchdog") + runtime_components = [ + ("ggml", "libggml.so", "4", "runtime:ggml", None), + ("ggml-base", "libggml-base.so", "5", "ggml", "a" * 40), + ("ggml-hip", "libggml-hip.so", "6", "selected-backend", None), + ("llama", "libllama.so", "7", "llama", None), + ("llama-common", "libllama-common.so", "9", "build-info", "a" * 40), + ] + runtime_libraries = [ + { + "component": component, + "filename": filename, + "path": f"/home/repo/build/lib/{filename}", + "sha256": digest * 64, + "role": role, + "revision": revision, + } + for component, filename, digest, role, revision in runtime_components + ] + runtime_receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "a" * 40, + "profile": "sibling-lib", + "components": [ + { + "component": component, + "filename": filename, + "sha256": digest * 64, + "revision": revision, + } + for component, filename, digest, _role, revision in runtime_components + ], + } result = { "runtime": runtime, "revision": trace.DS4_REVISION if is_ds4 else "a" * 40, + "authorization": trace.execution_authorization( + lane=trace.ORACLE_LANE if is_ds4 else trace.CANDIDATE_LANE, + challenge=TEST_CHALLENGE, + run_id=TEST_RUN_IDS[runtime], + issued_unix=TEST_AUTH_ISSUED, + expires_unix=TEST_AUTH_EXPIRES, + ), "build": ( { "compiler": "clang", @@ -394,20 +442,21 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "target": "arm64-apple-darwin", "path": "/home/repo/build/bin/llama-deepseek-v41-trace", "sha256": "3" * 64, - "runtime_libraries": sorted([ - { - "roles": [role], - "path": f"/home/repo/build/bin/{name}", - "sha256": digest * 64, - "revision": "a" * 40 if role in {"build-info", "ggml"} else None, - } - for role, name, digest in ( - ("build-info", "libllama-common.so", "4"), - ("llama", "libllama.so", "5"), - ("ggml", "libggml.so", "6"), - ("selected-backend", "libggml-hip.so", "7"), - ) - ], key=lambda library: library["path"]), + "runtime_profile": { + "name": "sibling-lib", + "components": [component for component, *_rest in runtime_components], + "selected_backend_component": "ggml-hip", + }, + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(runtime_receipt).encode("ascii")), + "runtime_libraries": sorted(runtime_libraries, key=lambda library: library["path"]), + "runtime_libraries_post": sorted( + copy.deepcopy(runtime_libraries), key=lambda library: library["path"]), + "runtime_module_monitor": { + "mechanism": "pre-post-snapshot", + "checked_after_trace": True, + "project_additions": [], + }, } ), "model": { @@ -519,7 +568,11 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "executable_path": result["build"]["path"], "executable_sha256": "3" * 64, "runtime_libraries_sha256": trace.sha256_bytes( - trace.canonical_json(result["build"]["runtime_libraries"]).encode("ascii")), + trace.canonical_json({ + "pre": result["build"]["runtime_libraries"], + "post": result["build"]["runtime_libraries_post"], + }).encode("ascii")), + "runtime_receipt_sha256": result["build"]["runtime_receipt_sha256"], } else: result["host"] = dict(DS4_HOST_ATTESTATION) @@ -787,12 +840,533 @@ def replace_event_blob( class TraceFormatTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls._signing_directory = tempfile.TemporaryDirectory() + cls.ssh_keygen = trace.trusted_ssh_keygen_path() + cls.signing_keys = {} + cls.signer_principals = { + "llama.cpp": "dsv41-test-candidate", + "ds4": "dsv41-test-oracle", + } + cls.test_signers = {} + for runtime, principal in cls.signer_principals.items(): + signing_key = Path(cls._signing_directory.name) / f"{runtime.replace('.', '-')}-key" + subprocess.run( + [ + str(cls.ssh_keygen), + "-q", + "-t", "ed25519", + "-N", "", + "-f", str(signing_key), + ], + check=True, + ) + signing_key.chmod(0o600) + public_key = subprocess.check_output( + [str(cls.ssh_keygen), "-y", "-f", str(signing_key)], + text=True, + ).strip() + public_key = " ".join(public_key.split()[:2]) + lane = trace.ORACLE_LANE if runtime == "ds4" else trace.CANDIDATE_LANE + profile = "apple-metal" if runtime == "ds4" else "sibling-lib" + cls.signing_keys[runtime] = signing_key + cls.test_signers[principal] = { + "public_key": public_key, + "lane": lane, + "runtime": runtime, + "runtime_profile": profile, + } + cls.signing_key = cls.signing_keys["llama.cpp"] + cls.signer_principal = cls.signer_principals["llama.cpp"] + cls.verifier = cls._verifier_for_runtime("llama.cpp") + cls._trace_bundle_class = trace.TraceBundle + + @classmethod + def _verifier_for_runtime( + cls, + runtime: str, + *, + expected_challenge: str = TEST_CHALLENGE, + expected_run_id: str | None = None, + verification_unix: int | None = None, + seen_run_ids: set[str] | None = None) -> trace.TraceVerifier: + principal = cls.signer_principals[runtime] + policy = cls.test_signers[principal] + return trace.TraceVerifier.for_tests( + principal, + policy["public_key"], + lane=policy["lane"], + runtime=runtime, + runtime_profile=policy["runtime_profile"], + expected_challenge=expected_challenge, + expected_run_id=expected_run_id or TEST_RUN_IDS[runtime], + verification_unix=verification_unix or int(time.time()), + ssh_keygen=cls.ssh_keygen, + seen_run_ids=seen_run_ids, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls._signing_directory.cleanup() + def setUp(self) -> None: self._require_nvme_path = preflight.require_nvme_path preflight.require_nvme_path = lambda path, label, **kwargs: preflight.resolved(path) + self._trace_bundle_symbol = trace.TraceBundle + + def test_bundle(root: Path, verify_blobs: bool = True, **_kwargs: object) -> object: + self._prune_fixture_extras(Path(root)) + signature = Path(root) / trace.SIGNATURE_NAME + if signature.exists() or signature.is_symlink(): + signature.unlink() + manifest_record = trace.strict_json_loads( + (Path(root) / trace.MANIFEST_NAME).read_text(encoding="ascii")) + runtime = manifest_record["runtime"] + principal = self.signer_principals[runtime] + authorization = manifest_record["authorization"] + trace.seal_bundle( + Path(root), + private_key=self.signing_keys[runtime], + principal=principal, + expected_lane=authorization["lane"], + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + return self._trace_bundle_class( + Path(root), + verify_blobs, + verifier=self._verifier_for_runtime( + runtime, + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + ), + ) + + trace.TraceBundle = test_bundle def tearDown(self) -> None: preflight.require_nvme_path = self._require_nvme_path + trace.TraceBundle = self._trace_bundle_symbol + + def _seal_test_bundle(self, root: Path) -> str: + signature = root / trace.SIGNATURE_NAME + if signature.exists() or signature.is_symlink(): + signature.unlink() + manifest_record = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + runtime = manifest_record["runtime"] + principal = self.signer_principals[runtime] + authorization = manifest_record["authorization"] + return trace.seal_bundle( + root, + private_key=self.signing_keys[runtime], + principal=principal, + expected_lane=authorization["lane"], + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + def _prune_fixture_extras(self, root: Path) -> None: + manifest_path = root / trace.MANIFEST_NAME + events_path = root / trace.EVENTS_NAME + if not manifest_path.is_file() or not events_path.is_file(): + return + try: + manifest_record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + events = [ + trace.strict_json_loads(line) + for line in events_path.read_text(encoding="ascii").splitlines() + ] + except (OSError, UnicodeError, trace.TraceError): + return + expected = {trace.MANIFEST_NAME, trace.EVENTS_NAME} + if isinstance(manifest_record, dict): + prompt = manifest_record.get("prompt") + provenance = prompt.get("provenance") if isinstance(prompt, dict) else None + if isinstance(provenance, dict) and isinstance(provenance.get("path"), str): + expected.add(provenance["path"]) + audits = manifest_record.get("audits") + if isinstance(audits, dict): + for phase in audits.values(): + if not isinstance(phase, dict): + continue + for reference in phase.values(): + if not isinstance(reference, dict) or not isinstance(reference.get("path"), str): + continue + expected.add(reference["path"]) + audit_path = root / reference["path"] + if not audit_path.is_file(): + continue + try: + audit = trace.strict_json_loads(audit_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, trace.TraceError): + continue + nested = audit.get("data", {}).get("audit") if isinstance(audit, dict) else None + if isinstance(nested, dict) and isinstance(nested.get("path"), str): + expected.add(nested["path"]) + for event in events: + if isinstance(event, dict) and isinstance(event.get("blob"), str): + expected.add(event["blob"]) + for path in root.rglob("*"): + if path.is_file() and path.relative_to(root).as_posix() not in expected | {trace.SIGNATURE_NAME}: + path.unlink() + + def _read_sealed_bundle(self, root: Path) -> object: + return self._trace_bundle_class(root, verifier=self.verifier) + + def test_seal_requires_external_trust_and_fixed_verifier(self) -> None: + self.assertEqual(trace.APPROVED_TRACE_SIGNERS, {}) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(root) + self._read_sealed_bundle(root) + with self.assertRaisesRegex(trace.TraceError, "external signer, lane, challenge, and run ID"): + self._trace_bundle_class(root) + with self.assertRaisesRegex(trace.TraceError, "not approved"): + self._trace_bundle_class( + root, + signer_principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + ) + candidate_policy = self.test_signers[self.signer_principal] + unknown = trace.TraceVerifier.for_tests( + "unknown", + candidate_policy["public_key"], + lane=trace.CANDIDATE_LANE, + runtime="llama.cpp", + runtime_profile="sibling-lib", + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + verification_unix=int(time.time()), + ) + with self.assertRaisesRegex(trace.TraceError, "externally expected signer"): + self._trace_bundle_class(root, verifier=unknown) + + fake_directory = Path(temp) / "fake-bin" + fake_directory.mkdir() + fake = fake_directory / "ssh-keygen" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") + fake.chmod(0o755) + with mock.patch.dict(os.environ, {"PATH": str(fake_directory)}): + self._read_sealed_bundle(root) + substituted = Path(temp) / "substituted-ssh-keygen" + substituted.symlink_to(fake) + verifier = trace.TraceVerifier.for_tests( + self.signer_principal, + candidate_policy["public_key"], + lane=trace.CANDIDATE_LANE, + runtime="llama.cpp", + runtime_profile="sibling-lib", + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + verification_unix=int(time.time()), + ssh_keygen=substituted, + ) + with self.assertRaisesRegex(trace.TraceError, "non-symlink"): + self._trace_bundle_class(root, verifier=verifier) + + with self.assertRaisesRegex(trace.TraceError, "already exists"): + trace.seal_bundle( + root, + private_key=self.signing_key, + principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + other_key = Path(temp) / "other-key" + subprocess.run( + [ + str(self.ssh_keygen), + "-q", + "-t", "ed25519", + "-N", "", + "-f", str(other_key), + ], + check=True, + ) + other_key.chmod(0o600) + with self.assertRaisesRegex(trace.TraceError, "does not match"): + trace.validate_signing_identity( + other_key, + self.signer_principal, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + def test_seal_rejects_protected_bundle_mutations(self) -> None: + with tempfile.TemporaryDirectory() as temp: + baseline = Path(temp) / "baseline" + with trace.TraceBundleWriter(baseline, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(baseline) + self._read_sealed_bundle(baseline) + + mutations = { + "manifest": lambda root: (root / trace.MANIFEST_NAME).write_bytes( + (root / trace.MANIFEST_NAME).read_bytes().replace(b'"event_count":259', b'"event_count":258')), + "events": lambda root: (root / trace.EVENTS_NAME).write_bytes( + (root / trace.EVENTS_NAME).read_bytes().replace(b'"step":0', b'"step":1', 1)), + "audit": lambda root: next((root / "audits").rglob("*.json")).write_bytes(b"{}\n"), + "blob": lambda root: next((root / trace.BLOBS_DIR).iterdir()).write_bytes(b"changed"), + "provenance": lambda root: next((root / "provenance").iterdir()).write_bytes(b"{}\n"), + "added": lambda root: (root / "unexpected").write_bytes(b"x"), + "signature": lambda root: (root / trace.SIGNATURE_NAME).write_bytes( + (root / trace.SIGNATURE_NAME).read_bytes().replace(b"SSH SIGNATURE", b"SSH SIGNATURX", 1)), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + root = Path(temp) / name + shutil.copytree(baseline, root) + mutate(root) + with self.assertRaises(trace.TraceError): + self._read_sealed_bundle(root) + + def test_seal_binds_runtime_lane_challenge_and_run_id(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(root) + + with self.assertRaisesRegex(trace.TraceError, "external challenge"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime("llama.cpp", expected_challenge="e" * 64), + ) + with self.assertRaisesRegex(trace.TraceError, "external run ID"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime( + "llama.cpp", expected_run_id="strix-llama-other-run"), + ) + with self.assertRaisesRegex(trace.TraceError, "expected execution lane"): + wrong_lane = trace.TraceVerifier.for_tests( + self.signer_principal, + self.test_signers[self.signer_principal]["public_key"], + lane=trace.ORACLE_LANE, + runtime="ds4", + runtime_profile="apple-metal", + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["ds4"], + verification_unix=int(time.time()), + ssh_keygen=self.ssh_keygen, + ) + self._trace_bundle_class(root, verifier=wrong_lane) + + seen_run_ids: set[str] = set() + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime("llama.cpp", seen_run_ids=seen_run_ids), + ) + with self.assertRaisesRegex(trace.TraceError, "reused"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime("llama.cpp", seen_run_ids=seen_run_ids), + ) + + with self.assertRaisesRegex(trace.TraceError, "expired"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime( + "llama.cpp", verification_unix=TEST_AUTH_EXPIRES + 1), + ) + + def test_seal_rejects_nonportable_signed_paths(self) -> None: + invalid_paths = ( + "../escape.json", + "./provenance.json", + "audit//record.json", + r"audit\record.json", + "C:/audit/record.json", + "//server/share.json", + "%2e%2e/escape.json", + "audit/\x01.json", + "audit/\N{LATIN SMALL LETTER E WITH ACUTE}.json", + ) + for index, invalid in enumerate(invalid_paths): + with self.subTest(path=invalid), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + record = manifest() + record["prompt"]["provenance"]["path"] = invalid + with trace.TraceBundleWriter(root, record) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "trace path"): + self._seal_test_bundle(root) + + def test_signing_rejects_bundle_key_and_public_key_fallback(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + bundled_key = root / "private-key" + shutil.copyfile(self.signing_key, bundled_key) + bundled_key.chmod(0o600) + with self.assertRaisesRegex(trace.TraceError, "outside the bundle"): + trace.seal_bundle( + root, + private_key=bundled_key, + principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + public_key_path = self.signing_key.with_suffix(".pub") + public_key_path.chmod(0o600) + with self.assertRaisesRegex(trace.TraceError, "derive|match"): + trace.validate_signing_identity( + public_key_path, + self.signer_principal, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + with mock.patch.dict(os.environ, {"SSH_AUTH_SOCK": "/tmp/attacker-agent"}): + trace.validate_signing_identity( + self.signing_key, + self.signer_principal, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + def test_signer_policy_rejects_open_ssh_options_certificates_and_extra_fields(self) -> None: + policy = copy.deepcopy(self.test_signers[self.signer_principal]) + public_key = policy["public_key"] + invalid_keys = ( + f"cert-authority {public_key}", + public_key.replace("ssh-ed25519", "ssh-ed25519-cert-v01@openssh.com", 1), + f"{public_key} comment", + f"{public_key}\n{public_key}", + ) + for public_key_value in invalid_keys: + with self.subTest(public_key=public_key_value): + invalid_policy = copy.deepcopy(policy) + invalid_policy["public_key"] = public_key_value + with self.assertRaisesRegex(trace.TraceError, "OpenSSH Ed25519 key"): + trace.validate_signing_identity( + self.signing_key, + self.signer_principal, + trusted_signers={self.signer_principal: invalid_policy}, + ssh_keygen=self.ssh_keygen, + ) + with self.assertRaisesRegex(trace.TraceError, "principal is invalid"): + trace.validate_signing_identity( + self.signing_key, + "*", + trusted_signers={"*": policy}, + ssh_keygen=self.ssh_keygen, + ) + + def test_seal_rejects_noncanonical_duplicate_and_incomplete_files(self) -> None: + with tempfile.TemporaryDirectory() as temp: + baseline = Path(temp) / "baseline" + with trace.TraceBundleWriter(baseline, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(baseline) + + noncanonical = Path(temp) / "noncanonical" + shutil.copytree(baseline, noncanonical) + record = json.loads((noncanonical / trace.MANIFEST_NAME).read_text(encoding="ascii")) + (noncanonical / trace.MANIFEST_NAME).write_text( + json.dumps(record, sort_keys=True, indent=2) + "\n", + encoding="ascii", + ) + with self.assertRaisesRegex(trace.TraceError, "not canonical"): + self._read_sealed_bundle(noncanonical) + + duplicate = Path(temp) / "duplicate" + shutil.copytree(baseline, duplicate) + manifest_path = duplicate / trace.MANIFEST_NAME + data = manifest_path.read_text(encoding="ascii") + manifest_path.write_text(data.replace("{", '{"trace_format":"dsv41-trace",', 1), encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "duplicate JSON key"): + self._read_sealed_bundle(duplicate) + + truncated = Path(temp) / "truncated" + shutil.copytree(baseline, truncated) + events_path = truncated / trace.EVENTS_NAME + events_path.write_bytes(events_path.read_bytes()[:-1]) + with self.assertRaisesRegex(trace.TraceError, "truncated"): + self._read_sealed_bundle(truncated) + + missing = Path(temp) / "missing" + shutil.copytree(baseline, missing) + next((missing / trace.BLOBS_DIR).iterdir()).unlink() + with self.assertRaises(trace.TraceError): + self._read_sealed_bundle(missing) + + unsigned = Path(temp) / "unsigned" + shutil.copytree(baseline, unsigned) + (unsigned / trace.SIGNATURE_NAME).unlink() + with self.assertRaisesRegex(trace.TraceError, "bundle-signature"): + self._read_sealed_bundle(unsigned) + + for field, value in ( + ("namespace", "wrong-namespace"), + ("principal", "other-principal"), + ("format", "other-format"), + ("version", 2)): + with self.subTest(envelope_field=field): + root = Path(temp) / f"envelope-{field}" + shutil.copytree(baseline, root) + envelope_path = root / trace.SIGNATURE_NAME + envelope = json.loads(envelope_path.read_text(encoding="ascii")) + envelope[field] = value + envelope_path.write_text( + json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + with self.assertRaises(trace.TraceError): + self._read_sealed_bundle(root) + + def test_bundle_cannot_supply_signature_trust_inputs(self) -> None: + for field in ("signer_public_key", "signer_principal", "verifier_path", "allowed_signers"): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + record = manifest() + record[field] = "attacker-controlled" + with trace.TraceBundleWriter(root, record) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, f"unexpected {field}"): + trace.TraceBundle(root) + + def test_seal_rejects_hard_links_and_post_verify_swaps(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(root) + bundle = self._read_sealed_bundle(root) + event = bundle.events[0] + blob = root / event["blob"] + replacement = root / "replacement" + replacement.write_bytes(blob.read_bytes()) + os.replace(replacement, blob) + with self.assertRaisesRegex(trace.TraceError, "changed after signature verification"): + bundle.read_blob(event) + + if os.name != "nt": + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + blob = next((root / trace.BLOBS_DIR).iterdir()) + os.link(blob, Path(temp) / "hard-link") + with self.assertRaisesRegex(trace.TraceError, "hard linked"): + self._seal_test_bundle(root) def test_serialization_preserves_float_bits_and_hashes(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -1929,7 +2503,7 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: bad_manifest["audits"]["pre"]["watchdog"] = "watchdog.json" with trace.TraceBundleWriter(root, bad_manifest) as writer: add_required_events(writer) - with self.assertRaisesRegex(trace.TraceError, "watchdog audit reference"): + with self.assertRaisesRegex(trace.TraceError, "audit reference"): trace.TraceBundle(root) with tempfile.TemporaryDirectory() as temp: @@ -1938,7 +2512,7 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: with trace.TraceBundleWriter(root, trace_manifest) as writer: add_required_events(writer) (root / trace_manifest["audits"]["pre"]["memory"]["path"]).unlink() - with self.assertRaisesRegex(trace.TraceError, "pre memory audit evidence"): + with self.assertRaisesRegex(trace.TraceError, "missing or not regular"): trace.TraceBundle(root) with tempfile.TemporaryDirectory() as temp: @@ -1947,7 +2521,7 @@ def test_detects_truncated_and_corrupt_artifacts(self) -> None: with trace.TraceBundleWriter(root, trace_manifest) as writer: add_required_events(writer) (root / trace_manifest["audits"]["post"]["watchdog"]["path"]).unlink() - with self.assertRaisesRegex(trace.TraceError, "post watchdog audit evidence"): + with self.assertRaisesRegex(trace.TraceError, "missing or not regular"): trace.TraceBundle(root) def test_rejects_unpinned_ds4_revision(self) -> None: @@ -1982,55 +2556,84 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: library_manifest = manifest() library_manifest["build"]["runtime_libraries"][0]["sha256"] = "e" * 64 - cases.append((library_manifest, "candidate runtime library identities")) + library_manifest["build"]["runtime_libraries_post"][0]["sha256"] = "e" * 64 + cases.append((library_manifest, "runtime receipt SHA-256")) + + added_module_manifest = manifest() + added_module_manifest["build"]["runtime_module_monitor"]["project_additions"] = [ + {"path": "lib/libggml-injected.module"}] + cases.append((added_module_manifest, "runtime module addition during trace generation")) + + incomplete_monitor_manifest = manifest() + incomplete_monitor_manifest["build"]["runtime_module_monitor"]["checked_after_trace"] = False + cases.append((incomplete_monitor_manifest, "runtime module monitor did not complete")) library_path_manifest = manifest() library_path_manifest["build"]["runtime_libraries"][0]["path"] = ( "/home/repo/build/bin/../substituted/libllama-common.so") + library_path_manifest["build"]["runtime_libraries_post"][0]["path"] = ( + "/home/repo/build/bin/../substituted/libllama-common.so") cases.append((library_path_manifest, "runtime library path is not canonical")) external_library_manifest = manifest() external_library_manifest["build"]["runtime_libraries"][0]["path"] = ( "/Users/attacker/libggml-injected.dylib") external_library_manifest["build"]["runtime_libraries"].sort(key=lambda item: item["path"]) + external_library_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + external_library_manifest["build"]["runtime_libraries"]) cases.append((external_library_manifest, "outside the exporter runtime directory")) omitted_library_manifest = manifest() omitted_library_manifest["build"]["runtime_libraries"] = [ library for library in omitted_library_manifest["build"]["runtime_libraries"] - if "selected-backend" not in library["roles"] + if library["role"] != "selected-backend" ] - cases.append((omitted_library_manifest, "runtime library identities are incomplete")) + omitted_library_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + omitted_library_manifest["build"]["runtime_libraries"]) + cases.append((omitted_library_manifest, "set differs from the runtime profile")) duplicate_path_manifest = manifest() duplicate_path_manifest["build"]["runtime_libraries"][1]["path"] = ( duplicate_path_manifest["build"]["runtime_libraries"][0]["path"]) + duplicate_path_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_path_manifest["build"]["runtime_libraries"]) cases.append((duplicate_path_manifest, "runtime library path is duplicated")) duplicate_role_manifest = manifest() - duplicate_role_manifest["build"]["runtime_libraries"][1]["roles"] = ( - duplicate_role_manifest["build"]["runtime_libraries"][0]["roles"]) - cases.append((duplicate_role_manifest, "runtime library role is duplicated")) + duplicate_role_manifest["build"]["runtime_libraries"][1]["role"] = ( + duplicate_role_manifest["build"]["runtime_libraries"][0]["role"]) + duplicate_role_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_role_manifest["build"]["runtime_libraries"]) + cases.append((duplicate_role_manifest, "runtime library role is invalid")) - unsorted_role_manifest = manifest() - unsorted_role_manifest["build"]["runtime_libraries"][0]["roles"] = ["llama", "build-info"] - cases.append((unsorted_role_manifest, "runtime library roles are not sorted")) + unknown_component_manifest = manifest() + unknown_component_manifest["build"]["runtime_libraries"][0]["component"] = "ggml-injected" + unknown_component_manifest["build"]["runtime_libraries_post"][0]["component"] = "ggml-injected" + cases.append((unknown_component_manifest, "runtime library component is invalid")) revision_library_manifest = manifest() revision_library = next( library for library in revision_library_manifest["build"]["runtime_libraries"] - if "build-info" in library["roles"]) + if library["role"] == "build-info") revision_library["revision"] = "b" * 40 + next( + library + for library in revision_library_manifest["build"]["runtime_libraries_post"] + if library["role"] == "build-info")["revision"] = "b" * 40 cases.append((revision_library_manifest, "runtime library revision is invalid")) unexpected_revision_manifest = manifest() unexpected_revision = next( library for library in unexpected_revision_manifest["build"]["runtime_libraries"] - if not set(library["roles"]) & {"build-info", "ggml"}) + if library["role"].startswith("runtime:")) unexpected_revision["revision"] = "a" * 40 + next( + library + for library in unexpected_revision_manifest["build"]["runtime_libraries_post"] + if library["role"].startswith("runtime:"))["revision"] = "a" * 40 cases.append((unexpected_revision_manifest, "runtime library revision is unexpected")) for trace_manifest, message in cases: @@ -2074,7 +2677,7 @@ def test_accepts_truthful_metal_vs_strix_bundles(self) -> None: def test_rejects_cross_runtime_attestation_substitution(self) -> None: for runtime, accelerator, message in ( - ("ds4", ACCELERATOR_ATTESTATION, "ds4 accelerator attestation fields"), + ("ds4", ACCELERATOR_ATTESTATION, "runtime profile"), ("llama.cpp", METAL_ACCELERATOR_ATTESTATION, "llama.cpp accelerator attestation fields")): with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" @@ -2189,7 +2792,7 @@ def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: trace_manifest["accelerator"]["runtime_kind"] = "unknown" with trace.TraceBundleWriter(root, trace_manifest) as writer: add_required_events(writer) - with self.assertRaisesRegex(trace.TraceError, "runtime_kind mismatch"): + with self.assertRaisesRegex(trace.TraceError, "runtime profile"): trace.TraceBundle(root) with tempfile.TemporaryDirectory() as temp: @@ -2198,7 +2801,7 @@ def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: del trace_manifest["accelerator"]["runtime_kind"] with trace.TraceBundleWriter(root, trace_manifest) as writer: add_required_events(writer) - with self.assertRaisesRegex(trace.TraceError, "fields are invalid"): + with self.assertRaisesRegex(trace.TraceError, "runtime profile"): trace.TraceBundle(root) with tempfile.TemporaryDirectory() as temp: @@ -2278,13 +2881,30 @@ def test_native_complete_manifest_writer_validates(self) -> None: self.assertEqual(native["revision"], revision) self.assertEqual(native["build"]["path"], str(manifest_binary.resolve())) self.assertIn("test-only manifest harness", native["build"]["info"]) + self.assertEqual(native["accelerator"]["runtime_kind"], "native-test") + self.assertEqual(native["accelerator"]["device_type"], "cpu") + self.assertIs(native["accelerator"]["test_only"], True) + components = native["build"]["runtime_profile"]["components"] + self.assertEqual(components, sorted(components)) + self.assertEqual( + native["build"]["runtime_libraries_post"], + native["build"]["runtime_libraries"], + ) self.assertEqual( + native["build"]["runtime_module_monitor"], { - role - for library in native["build"]["runtime_libraries"] - for role in library["roles"] + "mechanism": "dyld-add-image" if sys.platform == "darwin" else "pre-post-snapshot", + "checked_after_trace": True, + "project_additions": [], }, - {"build-info", "llama", "ggml", "selected-backend"}, + ) + self.assertEqual( + {library["component"] for library in native["build"]["runtime_libraries"]}, + set(components), + ) + self.assertEqual( + len({library["role"] for library in native["build"]["runtime_libraries"]}), + len(components), ) self.assertEqual( [library["path"] for library in native["build"]["runtime_libraries"]], @@ -2293,13 +2913,38 @@ def test_native_complete_manifest_writer_validates(self) -> None: for library in native["build"]["runtime_libraries"]: expected_revision = ( revision - if set(library["roles"]) & {"build-info", "ggml"} + if library["component"] in {"llama-common", "ggml-base"} else None ) self.assertEqual(library["revision"], expected_revision) + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": revision, + "profile": native["build"]["runtime_profile"]["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in native["build"]["runtime_libraries"] + ], + key=lambda item: item["component"], + ), + } + self.assertEqual( + native["build"]["runtime_receipt_sha256"], + trace.sha256_bytes(trace.canonical_json(receipt).encode("ascii")), + ) self.assertIsInstance(native["environment"]["command"], str) self.assertEqual(json.loads(native["environment"]["command"]), command) - self.assertIs(native["config"]["flash_attention"], True) + self.assertIs(native["config"]["flash_attention"], False) + self.assertEqual(native["config"]["runtime_kind"], "native-test") + self.assertEqual(native["config"]["device_type"], "cpu") + self.assertIs(native["config"]["test_only"], True) self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) attestation = fixture["candidate"] attestation["revision"] = revision @@ -2313,11 +2958,20 @@ def test_native_complete_manifest_writer_validates(self) -> None: manifest_binary, trace.sha256_file(manifest_binary), ) - with self.assertRaisesRegex(trace.TraceError, "missing candidate"): - trace.TraceBundle(root) + with self.assertRaisesRegex(trace.TraceError, "execution authorization is missing"): + trace.seal_bundle( + root, + private_key=self.signing_key, + principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) for protected_field in ( - "accelerator", "build", "candidate", "comparison", "config", + "accelerator", "authorization", "build", "candidate", "comparison", "config", "environment", "paths", "revision", "runtime", "storage_policy"): protected_input = dict(writer_input) protected_input[protected_field] = fixture.get(protected_field, {}) @@ -2357,83 +3011,183 @@ def test_runtime_build_validator_rejects_closure_substitution(self) -> None: exporter = binary_directory / "llama-deepseek-v41-trace" exporter.write_bytes(b"exporter") records = [] - for name, roles, content in ( - ("libggml-hip.so", ["selected-backend"], b"backend"), - ("libggml.so", ["ggml"], b"ggml"), - ("libllama-common.so", ["build-info"], b"build"), - ("libllama.so", ["llama"], b"llama"), - ("libggml-blas.so", [], b"blas")): + for component, name, role, content in ( + ("ggml", "libggml.so", "runtime:ggml", b"ggml"), + ("ggml-base", "libggml-base.so", "ggml", b"base"), + ("ggml-blas", "libggml-blas.so", "runtime:ggml-blas", b"blas"), + ("ggml-hip", "libggml-hip.so", "selected-backend", b"backend"), + ("llama", "libllama.so", "llama", b"llama"), + ("llama-common", "libllama-common.so", "build-info", b"build")): path = library_directory / name path.write_bytes(content) records.append({ + "component": component, + "filename": name, "path": str(path.resolve()), "sha256": trace.sha256_file(path), - "roles": roles, - "revision": "a" * 40 if set(roles) & {"build-info", "ggml"} else None, + "role": role, + "revision": "a" * 40 if component in {"llama-common", "ggml-base"} else None, }) records.sort(key=lambda record: record["path"]) + components = sorted(record["component"] for record in records) + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "a" * 40, + "profile": "sibling-lib", + "components": sorted( + [ + { + "component": record["component"], + "filename": record["filename"], + "sha256": record["sha256"], + "revision": record["revision"], + } + for record in records + ], + key=lambda item: item["component"], + ), + } build_manifest = { "revision": "a" * 40, "build": { "path": str(exporter.resolve()), "sha256": trace.sha256_file(exporter), "info": "test", + "runtime_profile": { + "name": "sibling-lib", + "components": components, + "selected_backend_component": "ggml-hip", + }, + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(receipt).encode("ascii")), "runtime_libraries": records, + "runtime_libraries_post": copy.deepcopy(records), + "runtime_module_monitor": { + "mechanism": "pre-post-snapshot", + "checked_after_trace": True, + "project_additions": [], + }, }, } - digest = run_llama.validate_runtime_build( + libraries_digest, receipt_digest = run_llama.validate_runtime_build( build_manifest, exporter=exporter, exporter_sha256=trace.sha256_file(exporter), candidate_revision="a" * 40, ) self.assertEqual( - digest, - trace.sha256_bytes(trace.canonical_json(records).encode("ascii")), + libraries_digest, + trace.sha256_bytes(trace.canonical_json({ + "pre": records, + "post": records, + }).encode("ascii")), ) + self.assertEqual(receipt_digest, build_manifest["build"]["runtime_receipt_sha256"]) cases = [] omitted = copy.deepcopy(build_manifest) omitted["build"]["runtime_libraries"] = [ library for library in omitted["build"]["runtime_libraries"] - if "selected-backend" not in library["roles"] + if library["role"] != "selected-backend" ] - cases.append((omitted, "identities are incomplete")) + omitted["build"]["runtime_libraries_post"] = copy.deepcopy( + omitted["build"]["runtime_libraries"]) + cases.append((omitted, "set differs from the runtime profile")) changed_hash = copy.deepcopy(build_manifest) changed_hash["build"]["runtime_libraries"][0]["sha256"] = "f" * 64 + changed_hash["build"]["runtime_libraries_post"][0]["sha256"] = "f" * 64 cases.append((changed_hash, "SHA-256 mismatch")) + changed_post = copy.deepcopy(build_manifest) + changed_post["build"]["runtime_libraries_post"][0]["sha256"] = "f" * 64 + cases.append((changed_post, "closure changed during trace generation")) + + added_module = copy.deepcopy(build_manifest) + added_module["build"]["runtime_module_monitor"]["project_additions"] = [ + {"path": "lib/libggml-injected.module"}] + cases.append((added_module, "runtime module addition during trace generation")) + + incomplete_monitor = copy.deepcopy(build_manifest) + incomplete_monitor["build"]["runtime_module_monitor"]["checked_after_trace"] = False + cases.append((incomplete_monitor, "runtime module monitor did not complete")) + changed_revision = copy.deepcopy(build_manifest) revision_record = next( library for library in changed_revision["build"]["runtime_libraries"] - if "build-info" in library["roles"]) + if library["role"] == "build-info") revision_record["revision"] = "b" * 40 + next( + library + for library in changed_revision["build"]["runtime_libraries_post"] + if library["role"] == "build-info")["revision"] = "b" * 40 cases.append((changed_revision, "revision mismatch")) duplicate_role = copy.deepcopy(build_manifest) role_records = duplicate_role["build"]["runtime_libraries"] - next(library for library in role_records if not library["roles"])["roles"] = ["llama"] + next(library for library in role_records if library["role"].startswith("runtime:"))["role"] = "llama" + duplicate_role["build"]["runtime_libraries_post"] = copy.deepcopy(role_records) cases.append((duplicate_role, "role is invalid")) - external = root / "external" / "libggml-injected.so" + duplicate_component = copy.deepcopy(build_manifest) + duplicate_component["build"]["runtime_libraries"][1]["component"] = ( + duplicate_component["build"]["runtime_libraries"][0]["component"]) + duplicate_component["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_component["build"]["runtime_libraries"]) + cases.append((duplicate_component, "component is invalid")) + + selected_backend = copy.deepcopy(build_manifest) + selected_backend["build"]["runtime_profile"]["selected_backend_component"] = "ggml-blas" + cases.append((selected_backend, "selected backend component is not ggml-hip")) + + filename = copy.deepcopy(build_manifest) + filename["build"]["runtime_libraries"][0]["filename"] = "other.so" + filename["build"]["runtime_libraries_post"] = copy.deepcopy( + filename["build"]["runtime_libraries"]) + cases.append((filename, "path differs from the exact runtime profile")) + + receipt_digest = copy.deepcopy(build_manifest) + receipt_digest["build"]["runtime_receipt_sha256"] = "f" * 64 + cases.append((receipt_digest, "runtime receipt SHA-256 mismatch")) + + external = root / "external" / "libggml-blas.so" external.parent.mkdir() - external.write_bytes(b"injected") + external.write_bytes(b"blas") external_manifest = copy.deepcopy(build_manifest) - external_manifest["build"]["runtime_libraries"].append({ - "path": str(external.resolve()), - "sha256": trace.sha256_file(external), - "roles": [], + external_record = next( + library + for library in external_manifest["build"]["runtime_libraries"] + if library["component"] == "ggml-blas") + external_record["path"] = str(external.resolve()) + external_manifest["build"]["runtime_libraries"].sort(key=lambda record: record["path"]) + external_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + external_manifest["build"]["runtime_libraries"]) + cases.append((external_manifest, "path differs from the exact runtime profile")) + + catalogued_not_profile = copy.deepcopy(build_manifest) + injected = library_directory / "libggml-injected.so" + injected.write_bytes(b"injected") + catalogued_not_profile["build"]["runtime_libraries"].append({ + "component": "ggml-injected", + "filename": injected.name, + "path": str(injected.resolve()), + "sha256": trace.sha256_file(injected), + "role": "runtime:ggml-injected", "revision": None, }) - external_manifest["build"]["runtime_libraries"].sort(key=lambda record: record["path"]) - cases.append((external_manifest, "outside the exporter runtime directory")) + catalogued_not_profile["build"]["runtime_libraries"].sort(key=lambda record: record["path"]) + catalogued_not_profile["build"]["runtime_libraries_post"] = copy.deepcopy( + catalogued_not_profile["build"]["runtime_libraries"]) + cases.append((catalogued_not_profile, "component is invalid")) duplicate_path = copy.deepcopy(build_manifest) duplicate_path["build"]["runtime_libraries"][1]["path"] = ( duplicate_path["build"]["runtime_libraries"][0]["path"]) + duplicate_path["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_path["build"]["runtime_libraries"]) cases.append((duplicate_path, "duplicated or unsorted")) for candidate, message in cases: @@ -2461,18 +3215,32 @@ def test_native_rejects_injected_project_library(self) -> None: with tempfile.TemporaryDirectory() as temp: external = Path(temp) / injected.name shutil.copy2(injected, external) - environment = dict(os.environ) variable = "DYLD_INSERT_LIBRARIES" if sys.platform == "darwin" else "LD_PRELOAD" - environment[variable] = str(external) - rejected = subprocess.run( - [str(binary.resolve()), "--version"], - check=False, - capture_output=True, - text=True, - env=environment, - ) - self.assertNotEqual(rejected.returncode, 0) - self.assertIn("outside the exporter runtime directory", rejected.stderr) + inside = binary.parent / "renamed-injected.module" + shutil.copy2(injected, inside) + try: + for name, injected_path in (("outside", external), ("renamed-inside", inside)): + with self.subTest(name=name): + environment = dict(os.environ) + environment[variable] = str(injected_path) + rejected = subprocess.run( + [str(binary.resolve()), "--version"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("forbids loader override", rejected.stderr) + finally: + inside.unlink(missing_ok=True) + + def test_python_runner_rejects_loader_overrides(self) -> None: + for variable in trace.FORBIDDEN_LOADER_ENVIRONMENT: + with self.subTest(variable=variable), mock.patch.dict( + os.environ, {variable: "/tmp/untrusted-runtime"}, clear=True): + with self.assertRaisesRegex(trace.TraceError, variable): + trace.reject_loader_overrides() def test_prompt_builder_result_becomes_strict_provenance(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -2536,7 +3304,7 @@ def run_builder(command, **_kwargs): def test_rejects_cross_runtime_and_unknown_audit_envelopes(self) -> None: for mutation, message in ( (lambda value: value["audits"].update({"unknown": {}}), "audit envelope"), - (lambda value: value["audits"]["pre"].update({"watchdog": {}}), "audit kinds")): + (lambda value: value["audits"]["pre"].update({"watchdog": {}}), "audit reference")): with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" trace_manifest = manifest("ds4") @@ -2777,6 +3545,12 @@ def test_unapproved_ds4_exporter_is_not_executed(self) -> None: "--corpus-name", "correctness-prose.txt", "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], "--prompt-provenance", str(root / "prompt.json"), + "--signer-principal", self.signer_principals["ds4"], + "--signing-key", str(self.signing_keys["ds4"]), + "--execution-challenge", TEST_CHALLENGE, + "--run-id", TEST_RUN_IDS["ds4"], + "--authorization-issued-unix", str(TEST_AUTH_ISSUED), + "--authorization-expires-unix", str(TEST_AUTH_EXPIRES), "--preflight-only", ] with mock.patch.object(sys, "argv", argv), mock.patch.object( @@ -3109,7 +3883,9 @@ def test_local_bringup_reports_do_not_claim_cross_runtime_pass(self) -> None: second = Path(temp) / "second" with trace.TraceBundleWriter(first, manifest("llama.cpp")) as writer: add_required_events(writer) - with trace.TraceBundleWriter(second, manifest("llama.cpp")) as writer: + second_manifest = manifest("llama.cpp") + second_manifest["authorization"]["run_id"] = "strix-llama-test-run-2" + with trace.TraceBundleWriter(second, second_manifest) as writer: add_required_events(writer) result = trace.local_report( trace.TraceBundle(first), @@ -3119,6 +3895,21 @@ def test_local_bringup_reports_do_not_claim_cross_runtime_pass(self) -> None: self.assertEqual(result["status"], "BRINGUP PASS") self.assertNotEqual(result["status"], "TARGET PASS") self.assertEqual(result["cross_runtime_status"], "INCOMPLETE") + with mock.patch("sys.stdout", new_callable=io.StringIO): + self.assertEqual( + trace.command_compare_local(Namespace( + mode="self-consistency", + left=first, + right=second, + left_signer_principal=self.signer_principal, + right_signer_principal=self.signer_principal, + execution_challenge=TEST_CHALLENGE, + left_run_id=TEST_RUN_IDS["llama.cpp"], + right_run_id="strix-llama-test-run-2", + report=None, + )), + 0, + ) def test_local_base_regression_requires_attested_oracle_revision(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -3128,10 +3919,36 @@ def test_local_base_regression_requires_attested_oracle_revision(self) -> None: base_manifest["revision"] = "b" * 40 base_manifest["candidate"]["revision"] = "b" * 40 for library in base_manifest["build"]["runtime_libraries"]: - if set(library["roles"]) & {"build-info", "ggml"}: + if library["component"] in {"llama-common", "ggml-base"}: library["revision"] = "b" * 40 + base_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + base_manifest["build"]["runtime_libraries"]) + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "b" * 40, + "profile": base_manifest["build"]["runtime_profile"]["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in base_manifest["build"]["runtime_libraries"] + ], + key=lambda item: item["component"], + ), + } + receipt_sha256 = trace.sha256_bytes(trace.canonical_json(receipt).encode("ascii")) + base_manifest["build"]["runtime_receipt_sha256"] = receipt_sha256 base_manifest["candidate"]["runtime_libraries_sha256"] = trace.sha256_bytes( - trace.canonical_json(base_manifest["build"]["runtime_libraries"]).encode("ascii")) + trace.canonical_json({ + "pre": base_manifest["build"]["runtime_libraries"], + "post": base_manifest["build"]["runtime_libraries_post"], + }).encode("ascii")) + base_manifest["candidate"]["runtime_receipt_sha256"] = receipt_sha256 with trace.TraceBundleWriter(base, base_manifest) as writer: add_required_events(writer) with trace.TraceBundleWriter(integrated, manifest("llama.cpp")) as writer: diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index 5fa172318537..983284d363ac 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -1,5 +1,13 @@ set(TARGET llama-deepseek-v41-trace) +set(DSV41_INSTALL_COMPONENT DeepSeekV41Trace) + +if(NOT BUILD_SHARED_LIBS) + message(STATUS "Skipping DeepSeek V4.1 trace exporter because BUILD_SHARED_LIBS is disabled") + return() +endif() + find_package(Git REQUIRED) +find_package(Python3 3.10 REQUIRED COMPONENTS Interpreter) execute_process( COMMAND ${GIT_EXECUTABLE} rev-parse HEAD WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} @@ -12,34 +20,155 @@ if(NOT DSV41_BUILD_REVISION_RESULT EQUAL 0 OR NOT DSV41_BUILD_REVISION MATCHES "^[0-9a-f]+$") message(FATAL_ERROR "DeepSeek V4.1 trace exporter requires the exact Git revision") endif() +set(DSV41_RECEIPT_TARGETS llama-common llama ggml ggml-base) +set(DSV41_BACKEND_TARGETS ${GGML_AVAILABLE_BACKENDS}) +list(SORT DSV41_BACKEND_TARGETS) +foreach(backend IN LISTS DSV41_BACKEND_TARGETS) + if(TARGET ${backend}) + list(APPEND DSV41_RECEIPT_TARGETS ${backend}) + endif() +endforeach() +list(REMOVE_DUPLICATES DSV41_RECEIPT_TARGETS) + +if(WIN32) + set(DSV41_RUNTIME_PROFILE co-located) +else() + set(DSV41_RUNTIME_PROFILE sibling-lib) + if(APPLE) + set(DSV41_LIBRARY_RPATH "@loader_path") + set(DSV41_EXECUTABLE_RPATH "@loader_path/../lib") + else() + set(DSV41_LIBRARY_RPATH "\$ORIGIN") + set(DSV41_EXECUTABLE_RPATH "\$ORIGIN/../lib") + endif() +endif() + +set(DSV41_RECEIPT_ARGUMENTS) +set(DSV41_RECEIPT_FILES) +foreach(component IN LISTS DSV41_RECEIPT_TARGETS) + if(NOT TARGET ${component}) + message(FATAL_ERROR "DeepSeek V4.1 receipt target is missing: ${component}") + endif() + get_target_property(component_type ${component} TYPE) + if(NOT component_type STREQUAL "SHARED_LIBRARY" AND NOT component_type STREQUAL "MODULE_LIBRARY") + message(FATAL_ERROR "DeepSeek V4.1 receipt target is not shared: ${component}") + endif() + if(component STREQUAL "llama-common" OR component STREQUAL "ggml-base") + set(component_revision "${DSV41_BUILD_REVISION}") + else() + set(component_revision "-") + endif() + list(APPEND DSV41_RECEIPT_ARGUMENTS + --entry "${component}" "$" "${component_revision}") + list(APPEND DSV41_RECEIPT_FILES "$") + if(NOT WIN32) + set_target_properties( + ${component} + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_LIBRARY_RPATH}") + if(APPLE) + set_target_properties( + ${component} + PROPERTIES + BUILD_WITH_INSTALL_NAME_DIR TRUE + INSTALL_NAME_DIR "@rpath") + endif() + endif() +endforeach() + +set(DSV41_RECEIPT_HEADER "${CMAKE_CURRENT_BINARY_DIR}/dsv41-runtime-receipt.h") +set(DSV41_RECEIPT_JSON "${CMAKE_CURRENT_BINARY_DIR}/dsv41-runtime-receipt.json") +add_custom_command( + OUTPUT "${DSV41_RECEIPT_HEADER}" "${DSV41_RECEIPT_JSON}" + COMMAND + ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/generate-runtime-receipt.py" + --output "${DSV41_RECEIPT_HEADER}" + --json-output "${DSV41_RECEIPT_JSON}" + --revision "${DSV41_BUILD_REVISION}" + --profile "${DSV41_RUNTIME_PROFILE}" + ${DSV41_RECEIPT_ARGUMENTS} + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/generate-runtime-receipt.py" + ${DSV41_RECEIPT_FILES} + VERBATIM) +add_custom_target( + dsv41-runtime-receipt + DEPENDS "${DSV41_RECEIPT_HEADER}" "${DSV41_RECEIPT_JSON}") +set_source_files_properties( + llama-trace.cpp + PROPERTIES OBJECT_DEPENDS "${DSV41_RECEIPT_HEADER}") + add_executable(${TARGET} llama-trace.cpp) +add_dependencies(${TARGET} dsv41-runtime-receipt) target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) -target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) target_compile_features(${TARGET} PRIVATE cxx_std_17) target_compile_definitions(${TARGET} PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") +if(NOT WIN32) + set_target_properties( + ${TARGET} + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_EXECUTABLE_RPATH}") +endif() set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) add_executable(${PROMPT_TARGET} prompt-builder.cpp) target_link_libraries(${PROMPT_TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${PROMPT_TARGET} PRIVATE cxx_std_17) +if(NOT WIN32) + set_target_properties( + ${PROMPT_TARGET} + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_EXECUTABLE_RPATH}") +endif() if(LLAMA_TOOLS_INSTALL) - install(TARGETS ${TARGET} ${PROMPT_TARGET} RUNTIME) + install( + TARGETS ${TARGET} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT}) + install( + TARGETS ${DSV41_RECEIPT_TARGETS} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + NAMELINK_COMPONENT ${DSV41_INSTALL_COMPONENT}) + install( + TARGETS ${PROMPT_TARGET} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT}) endif() if(LLAMA_BUILD_TESTS) set(MANIFEST_TEST_TARGET test-deepseek41-trace-manifest) add_executable(${MANIFEST_TEST_TARGET} llama-trace.cpp) + add_dependencies(${MANIFEST_TEST_TARGET} dsv41-runtime-receipt) target_link_libraries( ${MANIFEST_TEST_TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) - target_include_directories(${MANIFEST_TEST_TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_include_directories( + ${MANIFEST_TEST_TARGET} + PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) target_compile_features(${MANIFEST_TEST_TARGET} PRIVATE cxx_std_17) target_compile_definitions( ${MANIFEST_TEST_TARGET} PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}" + DSV41_SOURCE_ROOT="${CMAKE_SOURCE_DIR}" DSV41_MANIFEST_TEST_HARNESS=1) + if(NOT WIN32) + set_target_properties( + ${MANIFEST_TEST_TARGET} + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_EXECUTABLE_RPATH}") + endif() set(INJECTED_TEST_TARGET test-deepseek41-trace-injected) add_library(${INJECTED_TEST_TARGET} SHARED test-injected-library.cpp) @@ -49,11 +178,29 @@ if(LLAMA_BUILD_TESTS) PREFIX "" OUTPUT_NAME "libggml-injected" SUFFIX ".module") - set(HOST_TEST_TARGET test-deepseek41-trace-host) add_executable(${HOST_TEST_TARGET} test-host-attestation.cpp) target_link_libraries(${HOST_TEST_TARGET} PRIVATE ggml) target_compile_features(${HOST_TEST_TARGET} PRIVATE cxx_std_17) add_test(NAME ${HOST_TEST_TARGET} COMMAND $) set_property(TEST ${HOST_TEST_TARGET} PROPERTY LABELS main) + + if(LLAMA_TOOLS_INSTALL) + add_test( + NAME test-deepseek41-trace-install + COMMAND + ${CMAKE_COMMAND} + -DDSV41_BUILD_DIR=${CMAKE_BINARY_DIR} + -DDSV41_INSTALL_ROOT=${CMAKE_CURRENT_BINARY_DIR}/install-smoke + -DDSV41_COMPONENT=${DSV41_INSTALL_COMPONENT} + -DDSV41_EXECUTABLE=${TARGET}${CMAKE_EXECUTABLE_SUFFIX} + -DDSV41_SOURCE_EXECUTABLE=$ + -DDSV41_REVISION=${DSV41_BUILD_REVISION} + -DDSV41_CONFIG=$ + -DDSV41_PYTHON=${Python3_EXECUTABLE} + -DDSV41_RECEIPT=${DSV41_RECEIPT_JSON} + -DDSV41_VERIFY_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/verify-runtime-install.py + -P ${CMAKE_CURRENT_SOURCE_DIR}/test-install.cmake) + set_property(TEST test-deepseek41-trace-install PROPERTY LABELS main) + endif() endif() diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index ef29e3488131..301dbcaba10b 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -9,6 +9,11 @@ Each trace is a directory: - `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. - `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. - `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. +- `bundle-signature.json` contains the detached OpenSSH signature envelope for the exact bundle file set. + +Seal v1 signs `dsv41-trace-bundle-v1\n` followed by canonical JSON records for `manifest.json`, `events.jsonl`, every referenced event blob, every embedded audit, and prompt provenance. Each record binds its canonical relative path, byte count, and SHA-256. Validation rejects missing or added files, symlinks, hard links, nonregular files, path traversal, duplicate metadata references, noncanonical JSON or JSONL, duplicate JSON keys, truncation, concurrent replacement, and every post-seal mutation. + +Signer trust is external to the bundle. `APPROVED_TRACE_SIGNERS` maps one restricted ASCII principal to one exact OpenSSH Ed25519 public key, runtime lane, and runtime profile, and is intentionally empty until a separately authorized run. Candidate and oracle signers are not interchangeable. Every signed manifest also binds an externally supplied 256-bit challenge, lane-specific run ID, and bounded validity window. Validation requires those expected values from outside the bundle and rejects missing, mismatched, reused within a comparison, cross-lane, not-yet-valid, or expired authorization. The bundle cannot provide a public key, authoritative principal, verifier path, or allowed-signers file. Validation uses only `/usr/bin/ssh-keygen` on macOS and Linux or `C:\Windows\System32\OpenSSH\ssh-keygen.exe` on Windows, rejects symlinks and unsupported `-Y` implementations, clears SSH-agent influence, and never searches `PATH`. The private signing key must be an owned restrictive regular file outside the bundle and is never copied or logged. Tests use explicit test-only verifiers with ephemeral lane-specific keys; production validation does not trust those keys. Every manifest and memory audit declares that the expert cache and KV cache are memory-resident and that there are no external cache or state paths. Missing, substituted, or additional file-backed cache/state declarations fail closed. @@ -21,8 +26,14 @@ Internal tensors use raw ggml dimension order, and every dimension must be posit Validate or compare bundles: ```sh -python3 tools/deepseek-v41-trace/trace_format.py validate TRACE -python3 tools/deepseek-v41-trace/trace_format.py compare DS4_TRACE LLAMA_TRACE --report report.json +python3 tools/deepseek-v41-trace/trace_format.py validate TRACE \ + --signer-principal PRINCIPAL --lane strix-llama-candidate-v1 \ + --execution-challenge "$CHALLENGE" --run-id "$RUN_ID" +python3 tools/deepseek-v41-trace/trace_format.py compare DS4_TRACE LLAMA_TRACE \ + --left-signer-principal DS4_PRINCIPAL --right-signer-principal LLAMA_PRINCIPAL \ + --execution-challenge "$CHALLENGE" \ + --left-run-id "$DS4_RUN_ID" --right-run-id "$LLAMA_RUN_ID" \ + --report report.json ``` The first mismatch is reported by phase, decode step, exact token, layer, component, byte offset, flat element index, and per-token component element index. All required components use exact byte comparison. There is no tolerance mode. A ds4 bundle is invalid unless it reports revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. @@ -51,6 +62,16 @@ ROCm on `gfx1151` is the primary acceptance backend: ```sh HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ cmake -S . -B build-dsv41-trace-rocm \ + -DBUILD_SHARED_LIBS=ON \ + -DLLAMA_BUILD_TESTS=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_UI=OFF \ + -DLLAMA_USE_PREBUILT_UI=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DFETCHCONTENT_FULLY_DISCONNECTED=ON \ -DGGML_HIP=ON \ -DGPU_TARGETS=gfx1151 \ -DGGML_NATIVE=ON \ @@ -77,6 +98,8 @@ build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o CPY Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly, invokes the exact exporter for a pre-allocation device attestation, and rejects the run unless the backend PCI identity maps to exactly one KFD node reporting `gfx1151`. The native exporter repeats the query before model allocation and verifies that the loaded model still uses the same device. +Static repository builds skip this shared-library trace component instead of failing configuration. + Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the complete Strix matrix process group. The llama.cpp wrapper fails closed if this variable is absent or different, and every embedded Strix memory, swap, and watchdog audit records it. This Linux/ROCm setting is not an Apple Metal oracle requirement. ## Strix candidate execution gate @@ -119,9 +142,11 @@ The exporter is intentionally external to the canonical ds4 checkout. It must be The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. The content-addressed memory audit binds the preflight accelerator and storage attestations; the manifest binds the independently repeated native accelerator attestation. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, and repository path. The native exporter embeds the full 40-character candidate revision independently of dynamically loaded build-info, resolves its actual executable path, and enumerates every loaded `llama` and `ggml` project library through the platform loader. It canonicalizes, sorts, and hashes the complete closure, records exact revision evidence for revision-bearing modules, and rejects injected or loader-substituted project libraries outside the exporter `bin` and sibling `lib` roots. The launcher reopens and hashes every recorded module and rejects tracked or untracked checkout changes, prefix-only revision matches, omitted or duplicated roles, substituted executable or runtime-library paths, changed runtime-library bytes, test-only writer output, and any accelerator or loaded-model device mismatch. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, repository path, externally approved signer principal, and matching private signing key. The private key is never copied or logged. Its public half is derived with the fixed trusted `ssh-keygen` executable and must exactly match the source-controlled signer map before model execution. + +The native exporter embeds the full 40-character candidate revision independently of dynamically loaded build-info, resolves its actual executable path, and validates every loaded `llama`, `ggml`, and enabled backend project library against the build-generated component receipt and selected runtime profile. Component name, filename, canonical path, SHA-256, role, and exact revision-bearing identity must match, and the measured loaded set must equal the predeclared profile set both immediately before protected trace generation and after it completes. Both snapshots and the completed loader-monitor receipt are signed. macOS also monitors loader additions during the protected interval. Missing, duplicated, unclassified, outside-root, renamed inside-root injected, catalogued-but-not-profile, changed, or late-loaded project libraries fail closed. Linux enumerates loaded ELF objects rather than arbitrary memory mappings and accepts non-project ROCm dependencies only from a root-owned, non-writable `/opt/rocm` installation. Production launchers and the native exporter reject dynamic-loader and `GGML_BACKEND_PATH` overrides. The executable hash is bound separately by the signed manifest to avoid link-time hash circularity. -Production installs no manifest-writing or runtime-path probe option. With `LLAMA_BUILD_TESTS`, CMake builds the non-installed `test-deepseek41-trace-manifest` harness from the same native writer implementation. Its input accepts only model, prompt, audit, expected-coverage, and event-count data; protected runtime, accelerator, path, configuration, build, and environment fields are fixed internally, and the output carries a test-only build marker that `run_llama.py` refuses to bind as a candidate. +Production installs no manifest-writing or runtime-path probe option. With `LLAMA_BUILD_TESTS`, CMake builds the non-installed `test-deepseek41-trace-manifest` harness from the same native writer implementation. Its input accepts only model, prompt, audit, expected-coverage, and event-count data. Runtime, accelerator, path, configuration, build, and environment evidence comes from measured local state, and CPU or Darwin output is explicitly test-only and cannot satisfy production candidate or signer requirements. The trace install component places the exact receipt libraries beside the tools and sets `@loader_path/../lib` on macOS or `$ORIGIN/../lib` on ELF so installed `--version` needs no loader override. The install test also hashes every installed component and requires exact equality with the receipt embedded after the original library link; install-time rewriting or re-signing fails. `run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, and captures the llama.cpp side. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. @@ -151,6 +176,9 @@ CASE_ROOT="$RUN_ROOT/c32768" CANDIDATE_REV="$(git -C "$REPO" rev-parse HEAD)" BASE_REV= DIFF_SHA256="$(git -C "$REPO" diff --binary --no-ext-diff "$BASE_REV" "$CANDIDATE_REV" -- | sha256sum | awk '{print $1}')" +CHALLENGE=<64-lowercase-hex-execution-challenge> +AUTH_ISSUED= +AUTH_EXPIRES= mkdir -p "$CASE_ROOT/watchdog" cd "$REPO" @@ -175,6 +203,12 @@ HIP_LAUNCH_BLOCKING=1 python3 scripts/strix_memory_watchdog.py \ --candidate-revision "$CANDIDATE_REV" \ --base-revision "$BASE_REV" \ --candidate-diff-sha256 "$DIFF_SHA256" \ + --signer-principal "$LLAMA_SIGNER_PRINCIPAL" \ + --signing-key "$LLAMA_SIGNING_KEY" \ + --execution-challenge "$CHALLENGE" \ + --run-id-prefix "strix-llama-c32768" \ + --authorization-issued-unix "$AUTH_ISSUED" \ + --authorization-expires-unix "$AUTH_EXPIRES" \ --llama-only \ --contexts 32768 \ --ubatches 32 \ @@ -210,7 +244,13 @@ python3 tools/deepseek-v41-trace/run_ds4.py \ --context 32768 \ --decode-steps 8 \ --prefill-chunk 32 \ - --device Metal0 + --device Metal0 \ + --signer-principal "$DS4_SIGNER_PRINCIPAL" \ + --signing-key "$DS4_SIGNING_KEY" \ + --execution-challenge "$CHALLENGE" \ + --run-id "apple-ds4-prose-c32768-ub32" \ + --authorization-issued-unix "$AUTH_ISSUED" \ + --authorization-expires-unix "$AUTH_EXPIRES" ``` Compare the completed bundle with the matching Strix bundle using `trace_format.py compare`. Repeat for all four corpora before expanding the context matrix. @@ -253,10 +293,18 @@ For each case (`correctness-prose-c32768-ub32`, `correctness-code-c32768-ub32`, ```sh python3 tools/deepseek-v41-trace/trace_format.py compare-local self-consistency \ "$RUN_A/llama/$CASE" "$RUN_B/llama/$CASE" \ + --left-signer-principal "$SIGNER_PRINCIPAL" \ + --right-signer-principal "$SIGNER_PRINCIPAL" \ + --execution-challenge "$CHALLENGE" \ + --left-run-id "$RUN_A_ID" --right-run-id "$RUN_B_ID" \ --report "$REPORTS/$CASE-self.json" python3 tools/deepseek-v41-trace/trace_format.py compare-local base-regression \ "$BASE_RUN/llama/$CASE" "$RUN_A/llama/$CASE" \ + --left-signer-principal "$SIGNER_PRINCIPAL" \ + --right-signer-principal "$SIGNER_PRINCIPAL" \ + --execution-challenge "$CHALLENGE" \ + --left-run-id "$BASE_RUN_ID" --right-run-id "$RUN_A_ID" \ --report "$REPORTS/$CASE-base.json" ``` diff --git a/tools/deepseek-v41-trace/generate-runtime-receipt.py b/tools/deepseek-v41-trace/generate-runtime-receipt.py new file mode 100644 index 000000000000..9beb3024fab3 --- /dev/null +++ b/tools/deepseek-v41-trace/generate-runtime-receipt.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path + + +def quote(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--json-output", required=True, type=Path) + parser.add_argument("--revision", required=True) + parser.add_argument("--profile", required=True, choices=("co-located", "sibling-lib")) + parser.add_argument("--entry", action="append", nargs=3, default=[], metavar=("COMPONENT", "PATH", "REVISION")) + args = parser.parse_args() + + if re.fullmatch(r"[0-9a-f]{40}", args.revision) is None: + parser.error("revision must be an exact full Git revision") + if not args.entry: + parser.error("at least one receipt entry is required") + + entries = [] + components = set() + filenames = set() + digests = set() + for component, path_text, revision in args.entry: + path = Path(path_text).resolve(strict=True) + digest = sha256_file(path) + filename = path.name + if re.fullmatch(r"[a-z0-9-]+", component) is None: + parser.error(f"invalid component: {component}") + if re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None: + parser.error(f"invalid filename: {filename}") + if component in components: + parser.error(f"duplicate component: {component}") + if filename in filenames: + parser.error(f"duplicate filename: {filename}") + if digest in digests: + parser.error(f"duplicate SHA-256: {digest}") + if revision != "-" and revision != args.revision: + parser.error(f"invalid revision for {component}") + components.add(component) + filenames.add(filename) + digests.add(digest) + entries.append((component, filename, digest, "" if revision == "-" else revision)) + + entries.sort() + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": args.revision, + "profile": args.profile, + "components": [ + { + "component": component, + "filename": filename, + "sha256": digest, + "revision": revision or None, + } + for component, filename, digest, revision in entries + ], + } + receipt_sha256 = hashlib.sha256( + json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode("ascii") + ).hexdigest() + lines = [ + "#pragma once", + "", + "#include ", + "#include ", + "", + "namespace dsv41_runtime_receipt {", + "", + "struct entry {", + " const char * component;", + " const char * filename;", + " const char * sha256;", + " const char * revision;", + "};", + "", + f"inline constexpr const char * profile = {quote(args.profile)};", + f"inline constexpr const char * sha256 = {quote(receipt_sha256)};", + f"inline constexpr std::array entries = {{{{", + ] + for component, filename, digest, revision in entries: + lines.append( + " {" + + ", ".join((quote(component), quote(filename), quote(digest), quote(revision))) + + "}," + ) + lines.extend([ + "}};", + f"inline constexpr std::array components = {{{{", + ]) + for component, _, _, _ in entries: + lines.append(f" {quote(component)},") + lines.extend([ + "}};", + "", + "}", + "", + ]) + content = "\n".join(lines).encode("ascii") + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_suffix(args.output.suffix + ".tmp") + temporary.write_bytes(content) + os.replace(temporary, args.output) + receipt_content = ( + json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("ascii") + args.json_output.parent.mkdir(parents=True, exist_ok=True) + receipt_temporary = args.json_output.with_suffix(args.json_output.suffix + ".tmp") + receipt_temporary.write_bytes(receipt_content) + os.replace(receipt_temporary, args.json_output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index 5594c1b5e840..a01ac7ccedcf 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -10,11 +10,13 @@ extern "C" { #include "llama-ext.h" #include "host-attestation.h" #include "trace-components.h" +#include "dsv41-runtime-receipt.h" #include #include #include +#include #include #include #include @@ -27,6 +29,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -45,10 +48,10 @@ extern "C" { #endif #if defined(__linux__) #include +#include #include #include #include -#include #endif #endif @@ -57,7 +60,7 @@ extern "C" { #endif namespace fs = std::filesystem; -using json = nlohmann::ordered_json; +using json = nlohmann::json; static constexpr int TRACE_VERSION = 2; static constexpr const char * BUILD_REVISION = DSV41_BUILD_REVISION; @@ -185,20 +188,6 @@ static const void * function_address(T function) { return reinterpret_cast(reinterpret_cast(function)); } -static bool path_is_within(const fs::path & path, const fs::path & root) { - const fs::path relative = path.lexically_relative(root); - return !relative.empty() && *relative.begin() != ".."; -} - -static void require_runtime_module_location(const fs::path & executable, const fs::path & module) { - const fs::path binary_directory = executable.parent_path(); - const fs::path library_directory = binary_directory.parent_path() / "lib"; - if (module != executable && module.parent_path() != binary_directory && - !path_is_within(module, library_directory)) { - throw std::runtime_error("loaded runtime module is outside the exporter runtime directory: " + module.string()); - } -} - static bool is_project_runtime_library(const fs::path & path) { std::string name = path.filename().string(); std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { @@ -213,7 +202,117 @@ static bool is_project_runtime_library(const fs::path & path) { #endif } -static std::set loaded_project_runtime_libraries() { +static fs::path loaded_image_path(const fs::path & path) { + if (fs::exists(path)) { + return canonical_path(path, "loaded runtime module"); + } + if (!path.is_absolute()) { + throw std::runtime_error("loaded runtime module path is not absolute: " + path.string()); + } + return path.lexically_normal(); +} + +static bool is_trusted_system_runtime_path(const fs::path & path) { +#if defined(_WIN32) + std::vector buffer(32768); + const UINT size = GetSystemDirectoryW(buffer.data(), static_cast(buffer.size())); + if (size == 0 || size >= buffer.size()) { + throw std::runtime_error("cannot query the Windows system runtime directory"); + } + return path.parent_path() == canonical_path( + fs::path(std::wstring(buffer.data(), size)), "Windows system runtime directory"); +#elif defined(__APPLE__) + return path.string().rfind("/usr/lib/", 0) == 0 || + path.string().rfind("/System/Library/", 0) == 0; +#elif defined(__linux__) + const std::string value = path.string(); + std::error_code error; + const fs::path rocm_root = fs::canonical("/opt/rocm", error); + if (!error) { + const std::string prefix = rocm_root.string() + "/"; + if (value.rfind(prefix, 0) == 0) { + for (fs::path current = path; current != rocm_root.parent_path(); current = current.parent_path()) { + struct stat status = {}; + if (::stat(current.c_str(), &status) != 0 || status.st_uid != 0 || + (status.st_mode & (S_IWGRP | S_IWOTH)) != 0) { + return false; + } + if (current == rocm_root) { + return true; + } + } + } + } + return value.rfind("/lib/", 0) == 0 || + value.rfind("/lib64/", 0) == 0 || + value.rfind("/usr/lib/", 0) == 0 || + value.rfind("/usr/lib64/", 0) == 0; +#else + (void) path; + return false; +#endif +} + +static fs::path runtime_library_directory(const fs::path & executable); + +#if defined(__APPLE__) +static std::array protected_interval_images = {}; +static std::atomic protected_interval_image_count = 0; +static std::atomic protected_interval_active = false; + +static void record_loaded_image(const mach_header * header, intptr_t) { + if (!protected_interval_active.load(std::memory_order_relaxed)) { + return; + } + const size_t index = protected_interval_image_count.fetch_add(1, std::memory_order_relaxed); + if (index < protected_interval_images.size()) { + protected_interval_images[index] = header; + } +} + +static void begin_loader_monitor() { + static const bool registered = []() { + _dyld_register_func_for_add_image(record_loaded_image); + return true; + }(); + (void) registered; + protected_interval_image_count.store(0, std::memory_order_relaxed); + protected_interval_active.store(true, std::memory_order_release); +} + +static void end_loader_monitor(const fs::path & executable) { + protected_interval_active.store(false, std::memory_order_release); + const size_t count = protected_interval_image_count.load(std::memory_order_relaxed); + if (count > protected_interval_images.size()) { + throw std::runtime_error("too many loader image additions during protected trace generation"); + } + const fs::path library_directory = + canonical_path(runtime_library_directory(executable), "runtime library directory"); + for (size_t index = 0; index < count; ++index) { + Dl_info info = {}; + if (dladdr(protected_interval_images[index], &info) == 0 || + info.dli_fname == nullptr || info.dli_fname[0] == '\0') { + throw std::runtime_error("cannot identify loader image added during protected trace generation"); + } + const fs::path image = loaded_image_path(info.dli_fname); + if (image.parent_path() == executable.parent_path() || + image.parent_path() == library_directory || + is_project_runtime_library(image) || + !is_trusted_system_runtime_path(image)) { + throw std::runtime_error( + "runtime module was added during protected trace generation: " + image.string()); + } + } +} +#else +static void begin_loader_monitor() { +} + +static void end_loader_monitor(const fs::path &) { +} +#endif + +static std::set loaded_runtime_images(const fs::path & executable) { std::set result; #if defined(_WIN32) const HANDLE snapshot = CreateToolhelp32Snapshot( @@ -230,8 +329,9 @@ static std::set loaded_project_runtime_libraries() { } do { const fs::path reported_path = entry.szExePath; - if (is_project_runtime_library(reported_path)) { - result.insert(canonical_path(reported_path, "loaded runtime module")); + const fs::path path = loaded_image_path(reported_path); + if (path != executable) { + result.insert(path); } } while (Module32NextW(snapshot, &entry)); CloseHandle(snapshot); @@ -241,35 +341,102 @@ static std::set loaded_project_runtime_libraries() { const char * name = _dyld_get_image_name(index); if (name != nullptr && name[0] != '\0') { const fs::path reported_path = name; - if (is_project_runtime_library(reported_path)) { - result.insert(canonical_path(reported_path, "loaded runtime module")); + const fs::path path = loaded_image_path(reported_path); + if (path != executable) { + result.insert(path); } } } #elif defined(__linux__) - std::ifstream maps("/proc/self/maps"); - if (!maps) { - throw std::runtime_error("cannot enumerate loaded runtime modules"); - } - std::string line; - while (std::getline(maps, line)) { - const size_t path_start = line.find('/'); - if (path_start == std::string::npos) { - continue; + struct image_context { + const fs::path * executable; + std::set * result; + std::string error; + } context = {&executable, &result, {}}; + const auto callback = [](dl_phdr_info * info, size_t, void * data) { + image_context & context = *static_cast(data); + if (!context.error.empty() || info->dlpi_name == nullptr || info->dlpi_name[0] == '\0') { + return 0; } - const fs::path reported_path = line.substr(path_start); - if (is_project_runtime_library(reported_path)) { - result.insert(canonical_path(reported_path, "loaded runtime module")); + try { + const fs::path reported_path = info->dlpi_name; + if (!reported_path.is_absolute() && + (reported_path == "linux-vdso.so.1" || reported_path == "linux-gate.so.1")) { + return 0; + } + const fs::path path = loaded_image_path(reported_path); + if (path != *context.executable) { + context.result->insert(path); + } + } catch (const std::exception & error) { + context.error = error.what(); } + return 0; + }; + if (dl_iterate_phdr(callback, &context) < 0 || !context.error.empty()) { + throw std::runtime_error( + context.error.empty() ? "cannot enumerate loaded runtime modules" : context.error); } #endif return result; } +static bool is_lower_hex(const std::string & value, size_t length) { + return value.size() == length && std::all_of(value.begin(), value.end(), [](unsigned char character) { + return std::isdigit(character) || (character >= 'a' && character <= 'f'); + }); +} + +static fs::path runtime_library_directory(const fs::path & executable) { + const std::string profile = dsv41_runtime_receipt::profile; + if (profile == "co-located") { + return executable.parent_path(); + } + if (profile == "sibling-lib") { + return executable.parent_path().parent_path() / "lib"; + } + throw std::runtime_error("embedded runtime profile is invalid"); +} + +static void load_runtime_backends(const fs::path & executable) { +#if defined(GGML_BACKEND_DL) + const std::string directory = runtime_library_directory(executable).string(); + ggml_backend_load_all_from_path(directory.c_str()); +#else + (void) executable; + ggml_backend_load_all(); +#endif +} + +static void reject_loader_overrides() { + static const std::array names = { + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_IMAGE_SUFFIX", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "DYLD_ROOT_PATH", + "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", + "GGML_BACKEND_PATH", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + }; + for (const char * name : names) { + const char * value = std::getenv(name); + if (value != nullptr && value[0] != '\0') { + throw std::runtime_error(std::string("production trace execution forbids loader override: ") + name); + } + } +} + static json runtime_libraries_json( const fs::path & executable, ggml_backend_dev_t selected_device, - const std::string & revision) { + const std::string & revision, + const std::string & selected_backend_component) { if (selected_device == nullptr) { throw std::runtime_error("cannot bind a null selected backend device"); } @@ -277,39 +444,131 @@ static json runtime_libraries_json( if (selected_backend == nullptr) { throw std::runtime_error("selected backend device has no runtime registry"); } + if (dsv41_runtime_receipt::entries.size() != dsv41_runtime_receipt::components.size()) { + throw std::runtime_error("embedded runtime profile size differs from the receipt"); + } + + const fs::path library_directory = + canonical_path(runtime_library_directory(executable), "runtime library directory"); + std::map receipt_by_component; + std::map receipt_by_path; + std::set receipt_filenames; + std::set receipt_digests; + for (size_t index = 0; index < dsv41_runtime_receipt::entries.size(); ++index) { + const dsv41_runtime_receipt::entry & entry = dsv41_runtime_receipt::entries[index]; + const std::string component = entry.component; + const std::string filename = entry.filename; + const std::string digest = entry.sha256; + const std::string entry_revision = entry.revision; + if (component.empty() || filename.empty() || fs::path(filename).filename() != filename || + !is_lower_hex(digest, 64) || + !receipt_by_component.emplace(component, &entry).second || + !receipt_filenames.insert(filename).second || + !receipt_digests.insert(digest).second) { + throw std::runtime_error("embedded runtime receipt is not canonical and unique"); + } + if (component != dsv41_runtime_receipt::components[index]) { + throw std::runtime_error("embedded runtime profile differs from the receipt"); + } + const bool revision_bearing = component == "llama-common" || component == "ggml-base"; + if ((revision_bearing && entry_revision != revision) || + (!revision_bearing && !entry_revision.empty()) || + (!entry_revision.empty() && !is_lower_hex(entry_revision, 40))) { + throw std::runtime_error("embedded runtime receipt revision is invalid for " + component); + } + const fs::path expected_path = + canonical_path(library_directory / filename, "receipt runtime module"); + if (!receipt_by_path.emplace(expected_path, &entry).second) { + throw std::runtime_error("embedded runtime receipt path is duplicated"); + } + } + const fs::path build_info_module = module_path(function_address(&llama_commit)); const fs::path llama_module = module_path(function_address(&llama_model_load_from_file)); const fs::path ggml_module = module_path(function_address(&ggml_init)); const fs::path selected_backend_module = module_path(selected_backend); - std::set libraries = loaded_project_runtime_libraries(); - libraries.insert(build_info_module); - libraries.insert(llama_module); - libraries.insert(ggml_module); - libraries.insert(selected_backend_module); + const std::set images = loaded_runtime_images(executable); + std::set libraries; + const fs::path binary_directory = executable.parent_path(); + for (const fs::path & image : images) { + const bool in_runtime_root = + image.parent_path() == binary_directory || image.parent_path() == library_directory; + if (in_runtime_root || is_project_runtime_library(image)) { + libraries.insert(image); + } else if (!is_trusted_system_runtime_path(image)) { + throw std::runtime_error("loaded unclassified module outside trusted system roots: " + image.string()); + } + } + std::map loaded_by_component; + for (const fs::path & library : libraries) { + if (library.parent_path() != library_directory) { + throw std::runtime_error( + "loaded runtime module is outside the exporter runtime directory for the exact profile: " + + library.string()); + } + const auto receipt = receipt_by_path.find(library); + if (receipt == receipt_by_path.end()) { + throw std::runtime_error( + "loaded project runtime module is absent from the receipt: " + library.string()); + } + const std::string component = receipt->second->component; + if (!loaded_by_component.emplace(component, library).second) { + throw std::runtime_error("multiple loaded runtime modules map to receipt component " + component); + } + if (sha256_file(library) != receipt->second->sha256) { + throw std::runtime_error("loaded runtime module SHA-256 differs from the receipt: " + component); + } + } + if (loaded_by_component.size() != receipt_by_component.size()) { + for (const auto & item : receipt_by_component) { + if (loaded_by_component.count(item.first) == 0) { + throw std::runtime_error("receipt runtime component is not loaded: " + item.first); + } + } + throw std::runtime_error("loaded runtime component set differs from the receipt"); + } + + const std::array, 3> fixed_roles = {{ + {"llama-common", build_info_module}, + {"llama", llama_module}, + {"ggml-base", ggml_module}, + }}; + for (const auto & item : fixed_roles) { + const auto loaded = loaded_by_component.find(item.first); + if (loaded == loaded_by_component.end() || loaded->second != item.second) { + throw std::runtime_error( + "runtime symbol provider does not match receipt component " + std::string(item.first)); + } + } + const auto selected = loaded_by_component.find(selected_backend_component); + if (selected == loaded_by_component.end() || selected->second != selected_backend_module) { + throw std::runtime_error( + "selected backend module does not match receipt component " + selected_backend_component); + } json result = json::array(); for (const fs::path & library : libraries) { - require_runtime_module_location(executable, library); - std::vector roles; + const dsv41_runtime_receipt::entry & receipt = *receipt_by_path.at(library); + std::string role = "runtime:" + std::string(receipt.component); if (library == build_info_module) { - roles.push_back("build-info"); + role = "build-info"; } if (library == llama_module) { - roles.push_back("llama"); + role = "llama"; } if (library == ggml_module) { - roles.push_back("ggml"); + role = "ggml"; } if (library == selected_backend_module) { - roles.push_back("selected-backend"); + role = "selected-backend"; } - std::sort(roles.begin(), roles.end()); - const bool revision_bearing = library == build_info_module || library == ggml_module; result.push_back({ + {"component", receipt.component}, + {"filename", receipt.filename}, {"path", library.string()}, - {"sha256", sha256_file(library)}, - {"roles", std::move(roles)}, - {"revision", revision_bearing ? json(revision) : json(nullptr)}, + {"sha256", receipt.sha256}, + {"role", std::move(role)}, + {"revision", receipt.revision[0] == '\0' ? json(nullptr) : json(receipt.revision)}, }); } return result; @@ -318,6 +577,7 @@ static json runtime_libraries_json( static json runtime_build_json( const fs::path & executable, ggml_backend_dev_t selected_device, + const std::string & selected_backend_component, char ** argv) { const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked exporter"); if (invoked_path != executable) { @@ -348,7 +608,23 @@ static json runtime_build_json( {"target", llama_build_target()}, {"path", executable.string()}, {"sha256", sha256_file(executable)}, - {"runtime_libraries", runtime_libraries_json(executable, selected_device, revision)}, + {"runtime_profile", { + {"name", dsv41_runtime_receipt::profile}, + {"components", dsv41_runtime_receipt::components}, + {"selected_backend_component", selected_backend_component}, + }}, + {"runtime_receipt_sha256", dsv41_runtime_receipt::sha256}, + {"runtime_libraries", runtime_libraries_json( + executable, selected_device, revision, selected_backend_component)}, + {"runtime_module_monitor", { +#if defined(__APPLE__) + {"mechanism", "dyld-add-image"}, +#else + {"mechanism", "pre-post-snapshot"}, +#endif + {"checked_after_trace", false}, + {"project_additions", json::array()}, + }}, }; } @@ -765,6 +1041,14 @@ class trace_writer { } } + void bind_runtime_post(json runtime_libraries) { + if (manifest["build"]["runtime_libraries"] != runtime_libraries) { + throw std::runtime_error("loaded runtime component set changed during protected trace generation"); + } + manifest["build"]["runtime_libraries_post"] = std::move(runtime_libraries); + manifest["build"]["runtime_module_monitor"]["checked_after_trace"] = true; + } + void finish() { if (has_error()) { throw std::runtime_error(error_message); @@ -885,6 +1169,18 @@ static std::string runtime_system_info(const common_params & params) { return platform + "; " + common_params_get_system_info(params); } +static std::string runtime_platform_name() { +#if defined(_WIN32) + return "windows"; +#elif defined(__APPLE__) + return "darwin"; +#elif defined(__linux__) + return "linux"; +#else + return "unknown"; +#endif +} + static json accelerator_json(const dsv41::accelerator_attestation & accelerator) { return { {"format", "dsv41-accelerator-attestation"}, @@ -934,15 +1230,23 @@ static json storage_policy_json() { }; } +struct native_manifest_evidence { + json accelerator; + json paths; + json config; +}; + static json complete_manifest( json input, + native_manifest_evidence evidence, const fs::path & executable, ggml_backend_dev_t selected_device, + const std::string & selected_backend_component, const std::string & system_info, int argc, char ** argv) { - static const std::array required = { - "model", "prompt", "accelerator", "paths", "config", "audits", "expected", "event_count", + static const std::array required = { + "model", "prompt", "audits", "expected", "event_count", }; if (!input.is_object()) { throw std::runtime_error("manifest writer input is not a JSON object"); @@ -964,13 +1268,13 @@ static json complete_manifest( {"trace_version", TRACE_VERSION}, {"runtime", "llama.cpp"}, {"revision", BUILD_REVISION}, - {"build", runtime_build_json(executable, selected_device, argv)}, + {"build", runtime_build_json(executable, selected_device, selected_backend_component, argv)}, {"model", std::move(input["model"])}, {"prompt", std::move(input["prompt"])}, - {"accelerator", std::move(input["accelerator"])}, - {"paths", std::move(input["paths"])}, + {"accelerator", std::move(evidence.accelerator)}, + {"paths", std::move(evidence.paths)}, {"storage_policy", storage_policy_json()}, - {"config", std::move(input["config"])}, + {"config", std::move(evidence.config)}, {"comparison", { {"tokens", "exact"}, {"engram_rows", "exact"}, @@ -1005,6 +1309,62 @@ static void write_manifest_file(const fs::path & path, const json & manifest) { } #if defined(DSV41_MANIFEST_TEST_HARNESS) +static native_manifest_evidence test_manifest_evidence( + const fs::path & input_path, + const fs::path & output_path, + ggml_backend_dev_t device) { + if (device == nullptr || ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_CPU) { + throw std::runtime_error("manifest writer test requires a local CPU device"); + } + ggml_backend_dev_props properties = {}; + ggml_backend_dev_get_props(device, &properties); + ggml_backend_reg_t backend = ggml_backend_dev_backend_reg(device); + if (backend == nullptr) { + throw std::runtime_error("manifest writer test CPU has no runtime registry"); + } + const fs::path input = canonical_path(input_path, "manifest writer test input"); + const fs::path output_parent = + canonical_path(fs::absolute(output_path).parent_path(), "manifest writer test output directory"); + const fs::path output = output_parent / output_path.filename(); + const fs::path repository = canonical_path(DSV41_SOURCE_ROOT, "manifest writer source root"); + return { + { + {"format", "dsv41-native-test-accelerator"}, + {"version", 1}, + {"runtime_kind", "native-test"}, + {"platform", runtime_platform_name()}, + {"backend", ggml_backend_reg_name(backend)}, + {"backend_device", properties.name == nullptr ? "" : properties.name}, + {"backend_description", properties.description == nullptr ? "" : properties.description}, + {"device_type", "cpu"}, + {"source", "ggml-runtime"}, + {"test_only", true}, + }, + { + {"format", "dsv41-native-test-paths"}, + {"version", 1}, + {"input", input.string()}, + {"output", output.string()}, + {"repository", repository.string()}, + {"temporary_directory", output_parent.string()}, + {"test_only", true}, + }, + { + {"format", "dsv41-native-test-config"}, + {"version", 1}, + {"runtime_kind", "native-test"}, + {"platform", runtime_platform_name()}, + {"device", properties.name == nullptr ? "" : properties.name}, + {"device_description", properties.description == nullptr ? "" : properties.description}, + {"device_type", "cpu"}, + {"build_target", llama_build_target()}, + {"flash_attention", false}, + {"gpu_layers", 0}, + {"test_only", true}, + }, + }; +} + static void write_manifest_probe( const fs::path & input_path, const fs::path & output_path, @@ -1030,73 +1390,28 @@ static void write_manifest_probe( throw std::runtime_error("manifest writer test input has unexpected field: " + item.key()); } } - input["accelerator"] = { - {"format", "dsv41-accelerator-attestation"}, - {"version", 2}, - {"runtime_kind", "strix-rocm"}, - {"platform", "linux"}, - {"backend", "ROCm"}, - {"backend_device", "ROCm0"}, - {"backend_description", "AMD Radeon Graphics"}, - {"pci_device_id", "0000:c1:00.0"}, - {"kfd_node", 1}, - {"gpu_id", 42}, - {"gfx_target_version", 110501}, - {"architecture", "gfx1151"}, - {"source", "linux-kfd-sysfs"}, - }; - input["paths"] = { - {"model", "/mnt/models/model.gguf"}, - {"prompt", "/home/prompt.txt"}, - {"output", "/home"}, - {"repository", "/home/repo"}, - {"temporary_directory", "/home/tmp"}, - }; - input["config"] = { - {"context", 3}, - {"batch", 2048}, - {"ubatch", 32}, - {"device", "ROCm0"}, - {"device_architecture", "gfx1151"}, - {"device_pci_id", "0000:c1:00.0"}, - {"decode_steps", 1}, - {"kv_type_k", "f16"}, - {"kv_type_v", "f16"}, - {"flash_attention", true}, - {"gpu_layers", 99}, - {"load_mode", 0}, - {"expert_cache_slots", 192}, - {"expert_cache_bytes", UINT64_C(76441190400)}, - {"tokenizer_add_bos", true}, - {"tokenizer_parse_special", true}, - {"deepseek41", { - {"layer_count", 40}, - {"vocab_size", 129280}, - {"engram_layers", {1, 14}}, - {"engram_rows_per_token", 24}, - {"expert_count", 384}, - {"experts_used", 6}, - {"candidate_source_layer", 20}, - {"candidate_topk_blocks", 2048}, - {"candidate_block_size", 8}, - {"index_top_k", 512}, - {"raw_attention_layers", {0, 1}}, - {"raw_attention_width", 128}, - {"candidate_propagation_layers", {24, 28, 32, 36}}, - }}, - }; common_init(); - ggml_backend_load_all(); + const fs::path executable = current_executable_path(); + load_runtime_backends(executable); ggml_backend_dev_t device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); - write_manifest_file( - output_path, - complete_manifest( - std::move(input), - current_executable_path(), - device, - "Linux model-free manifest writer test", - argc, - argv)); + json manifest = complete_manifest( + std::move(input), + test_manifest_evidence(input_path, output_path, device), + executable, + device, + "ggml-cpu", + runtime_platform_name() + " local CPU model-free manifest writer test", + argc, + argv); + begin_loader_monitor(); + end_loader_monitor(executable); + manifest["build"]["runtime_libraries_post"] = runtime_libraries_json( + executable, device, BUILD_REVISION, "ggml-cpu"); + if (manifest["build"]["runtime_libraries"] != manifest["build"]["runtime_libraries_post"]) { + throw std::runtime_error("loaded runtime component set changed during manifest writer test"); + } + manifest["build"]["runtime_module_monitor"]["checked_after_trace"] = true; + write_manifest_file(output_path, manifest); } int main(int argc, char ** argv) { @@ -1116,12 +1431,15 @@ int main(int argc, char ** argv) { int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { + reject_loader_overrides(); if (argc == 2 && std::string(argv[1]) == "--version") { common_init(); - ggml_backend_load_all(); + const fs::path executable = current_executable_path(); + load_runtime_backends(executable); const json build = runtime_build_json( - current_executable_path(), + executable, ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU), + "ggml-cpu", argv); std::cout << "version: deepseek-v41-trace (build " << build["number"] << ", commit " << BUILD_REVISION << ")\n"; @@ -1131,7 +1449,7 @@ int main(int argc, char ** argv) { } if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { common_init(); - ggml_backend_load_all(); + load_runtime_backends(current_executable_path()); const dsv41::accelerator_attestation accelerator = dsv41::require_gfx1151_device(ggml_backend_dev_by_name(argv[2])); std::cout << accelerator_json(accelerator).dump() << '\n'; @@ -1148,6 +1466,7 @@ int main(int argc, char ** argv) { params.warmup = false; params.ctx_shift = false; common_init(); + load_runtime_backends(current_executable_path()); if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_RESULTS)) { return 1; } @@ -1245,27 +1564,16 @@ int main(int argc, char ** argv) { all_layers[layer] = layer; } - json manifest_input = { - {"model", { - {"path", model_path.string()}, - {"architecture", "deepseek41"}, - {"byte_count", fs::file_size(model_path)}, - {"sha256", sha256_file(model_path)}, - }}, - {"prompt", { - {"path", prompt_path.string()}, - {"byte_count", prompt_bytes.size()}, - {"sha256", sha256_data(prompt_bytes.data(), prompt_bytes.size())}, - }}, - {"accelerator", accelerator_json(accelerator)}, - {"paths", { + native_manifest_evidence evidence = { + accelerator_json(accelerator), + { {"model", model_path.string()}, {"prompt", prompt_path.string()}, {"output", output_path.string()}, {"repository", audited_storage["repository"].value("resolved_path", "")}, {"temporary_directory", temporary_storage.resolved_path.string()}, - }}, - {"config", { + }, + { {"context", llama_n_ctx(ctx)}, {"batch", params.n_batch}, {"ubatch", params.n_ubatch}, @@ -1297,6 +1605,19 @@ int main(int argc, char ** argv) { {"raw_attention_width", 128}, {"candidate_propagation_layers", {24, 28, 32, 36}}, }}, + }, + }; + json manifest_input = { + {"model", { + {"path", model_path.string()}, + {"architecture", "deepseek41"}, + {"byte_count", fs::file_size(model_path)}, + {"sha256", sha256_file(model_path)}, + }}, + {"prompt", { + {"path", prompt_path.string()}, + {"byte_count", prompt_bytes.size()}, + {"sha256", sha256_data(prompt_bytes.data(), prompt_bytes.size())}, }}, {"audits", { {"memory", memory_audit}, @@ -1323,12 +1644,15 @@ int main(int argc, char ** argv) { }; json manifest = complete_manifest( std::move(manifest_input), + std::move(evidence), executable_path, params.devices[0], + "ggml-hip", runtime_system_info(params), argc, argv); + begin_loader_monitor(); trace_writer writer(output_path, std::move(manifest)); llama_set_eval_callback(ctx, trace_callback, &writer); @@ -1367,6 +1691,9 @@ int main(int argc, char ** argv) { #if defined(__linux__) validate_watchdog(watchdog_audit["data"]); #endif + end_loader_monitor(executable_path); + writer.bind_runtime_post(runtime_libraries_json( + executable_path, params.devices[0], BUILD_REVISION, "ggml-hip")); writer.finish(); llama_backend_free(); return 0; diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index e4dd78b84555..e89860b23444 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -23,16 +23,24 @@ from trace_format import ( ADMITTED_UBATCH, APPROVED_EXPORTERS, + APPROVED_TRACE_SIGNERS, CORPUS_SHA256, DS4_REVISION, MODEL_SHA256, NO_EXTERNAL_STATE_STORAGE, + ORACLE_LANE, TraceBundle, TraceError, + TraceVerifier, + bind_execution_authorization, canonical_json, + execution_authorization, + reject_loader_overrides, + seal_bundle, sha256_bytes, sha256_file, strict_json_loads, + validate_signing_identity, ) def git_output(checkout: Path, *args: str) -> str: @@ -271,6 +279,12 @@ def main() -> int: parser.add_argument("--decode-steps", type=int, default=8) parser.add_argument("--prefill-chunk", type=int, default=ADMITTED_UBATCH) parser.add_argument("--device", default="Metal0") + parser.add_argument("--signer-principal", required=True) + parser.add_argument("--signing-key", type=Path, required=True) + parser.add_argument("--execution-challenge", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--authorization-issued-unix", type=int, required=True) + parser.add_argument("--authorization-expires-unix", type=int, required=True) parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() @@ -279,6 +293,15 @@ def main() -> int: raise PreflightError( f"DeepSeek V4.1 correctness runs require admitted prefill chunk {ADMITTED_UBATCH}, " f"found {args.prefill_chunk}") + reject_loader_overrides() + authorization = execution_authorization( + lane=ORACLE_LANE, + challenge=args.execution_challenge, + run_id=args.run_id, + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + ) + output = resolved(args.output) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") exporter = resolved(args.exporter) @@ -289,7 +312,12 @@ def main() -> int: raise PreflightError( f"trace exporter SHA-256 mismatch: expected {args.exporter_sha256}, found {exporter_sha256}") verify_exporter_approval(exporter_sha256) - output = resolved(args.output) + validate_signing_identity( + args.signing_key, + args.signer_principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + forbidden_root=output, + ) command = [ str(exporter), "--model", str(resolved(args.model)), @@ -346,7 +374,25 @@ def main() -> int: bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) bind_oracle_attestation(output, preflight_audit, accelerator, command) - bundle = TraceBundle(output) + bind_execution_authorization(output, authorization) + seal_bundle( + output, + private_key=args.signing_key, + principal=args.signer_principal, + expected_lane=ORACLE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + trusted_signers=APPROVED_TRACE_SIGNERS, + ) + bundle = TraceBundle( + output, + verifier=TraceVerifier.production( + args.signer_principal, + expected_lane=ORACLE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + ), + ) if bundle.manifest.get("runtime") != "ds4": raise PreflightError("ds4 exporter wrote a non-ds4 trace") if bundle.manifest.get("revision") != DS4_REVISION: diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 23fb80aececf..f03bf0431850 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -25,6 +25,8 @@ from trace_format import ( ADMITTED_BATCH, ADMITTED_UBATCH, + APPROVED_TRACE_SIGNERS, + CANDIDATE_LANE, CORPUS_SHA256, MODEL_SHA256, REPOSITORY, @@ -33,10 +35,16 @@ REQUIRED_EXPERT_SLOTS, TraceBundle, TraceError, + TraceVerifier, + bind_execution_authorization, canonical_json, + execution_authorization, + reject_loader_overrides, + seal_bundle, sha256_bytes, sha256_file, strict_json_loads, + validate_signing_identity, ) @@ -100,20 +108,12 @@ def candidate_attestation( } -def _path_is_within(path: Path, root: Path) -> bool: - try: - path.relative_to(root) - return True - except ValueError: - return False - - def validate_runtime_build( manifest: dict[str, object], *, exporter: Path, exporter_sha256: str, - candidate_revision: str) -> str: + candidate_revision: str) -> tuple[str, str]: build = manifest.get("build") if not isinstance(build, dict): raise PreflightError("llama trace build identity is missing") @@ -129,27 +129,64 @@ def validate_runtime_build( libraries = build.get("runtime_libraries") if not isinstance(libraries, list) or not libraries: raise PreflightError("llama trace runtime library identities are missing") - expected_roles = {"build-info", "llama", "ggml", "selected-backend"} + post_libraries = build.get("runtime_libraries_post") + if post_libraries != libraries: + raise PreflightError("llama trace runtime library closure changed during trace generation") + module_monitor = build.get("runtime_module_monitor") + if not isinstance(module_monitor, dict) or set(module_monitor) != { + "mechanism", "checked_after_trace", "project_additions"}: + raise PreflightError("llama trace runtime module monitor is invalid") + if module_monitor["mechanism"] not in {"dyld-add-image", "pre-post-snapshot"}: + raise PreflightError("llama trace runtime module monitor mechanism is invalid") + if module_monitor["checked_after_trace"] is not True: + raise PreflightError("llama trace runtime module monitor did not complete") + if module_monitor["project_additions"] != []: + raise PreflightError("llama trace records a runtime module addition during trace generation") + profile = build.get("runtime_profile") + if not isinstance(profile, dict) or set(profile) != { + "name", "components", "selected_backend_component"}: + raise PreflightError("llama trace runtime profile is invalid") + if profile.get("name") != "sibling-lib": + raise PreflightError("llama trace runtime profile is not the Linux sibling-lib profile") + components = profile.get("components") + if not isinstance(components, list) or components != sorted(components) or ( + len(components) != len(set(components))): + raise PreflightError("llama trace runtime profile components are invalid") + if not {"llama-common", "llama", "ggml", "ggml-base", "ggml-hip"}.issubset(set(components)): + raise PreflightError("llama trace runtime profile is missing required ROCm components") + selected_backend_component = profile.get("selected_backend_component") + if selected_backend_component != "ggml-hip": + raise PreflightError("llama trace selected backend component is not ggml-hip") roles = set() paths = set() + found_components = set() previous_path = None - binary_directory = exporter.parent - library_directory = binary_directory.parent / "lib" + library_directory = resolved(exporter.parent.parent / "lib") for library in libraries: - if not isinstance(library, dict): + if not isinstance(library, dict) or set(library) != { + "component", "filename", "path", "sha256", "role", "revision"}: raise PreflightError("llama trace runtime library identity is invalid") + component = library.get("component") + filename = library.get("filename") path_value = library.get("path") digest = library.get("sha256") - library_roles = library.get("roles") + role = library.get("role") revision = library.get("revision") - if not isinstance(library_roles, list) or library_roles != sorted(library_roles) or any( - role not in expected_roles for role in library_roles): - raise PreflightError("llama trace runtime library roles are invalid") - for role in library_roles: - if role in roles: - raise PreflightError("llama trace runtime library role is invalid") - roles.add(role) - if bool(set(library_roles) & {"build-info", "ggml"}): + if component not in components or component in found_components: + raise PreflightError("llama trace runtime library component is invalid") + found_components.add(component) + if not isinstance(filename, str) or Path(filename).name != filename: + raise PreflightError("llama trace runtime library filename is invalid") + expected_role = { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + selected_backend_component: "selected-backend", + }.get(component, f"runtime:{component}") + if role != expected_role or role in roles: + raise PreflightError("llama trace runtime library role is invalid") + roles.add(role) + if component in {"llama-common", "ggml-base"}: if revision != candidate_revision: raise PreflightError("llama trace runtime library revision mismatch") elif revision is not None: @@ -163,13 +200,35 @@ def validate_runtime_build( raise PreflightError("llama trace runtime library paths are duplicated or unsorted") paths.add(path) previous_path = path - if path != exporter and path.parent != binary_directory and not _path_is_within(path, library_directory): - raise PreflightError("llama trace runtime library is outside the exporter runtime directory") + if path.parent != library_directory or path.name != filename: + raise PreflightError("llama trace runtime library path differs from the exact runtime profile") if not path.is_file() or not isinstance(digest, str) or sha256_file(path) != digest: raise PreflightError("llama trace runtime library SHA-256 mismatch") - if roles != expected_roles: - raise PreflightError("llama trace runtime library identities are incomplete") - return sha256_bytes(canonical_json(libraries).encode("ascii")) + if found_components != set(components): + raise PreflightError("llama trace runtime library set differs from the runtime profile") + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": candidate_revision, + "profile": profile["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in libraries + ], + key=lambda item: item["component"], + ), + } + receipt_sha256 = sha256_bytes(canonical_json(receipt).encode("ascii")) + if build.get("runtime_receipt_sha256") != receipt_sha256: + raise PreflightError("llama trace runtime receipt SHA-256 mismatch") + closure = {"pre": libraries, "post": post_libraries} + return sha256_bytes(canonical_json(closure).encode("ascii")), receipt_sha256 def bind_candidate_attestation( @@ -186,12 +245,14 @@ def bind_candidate_attestation( if manifest.get("accelerator") != accelerator: raise PreflightError("llama trace accelerator attestation differs from the preflight query") bound_attestation = dict(attestation) - bound_attestation["runtime_libraries_sha256"] = validate_runtime_build( + libraries_sha256, receipt_sha256 = validate_runtime_build( manifest, exporter=exporter, exporter_sha256=exporter_sha256, candidate_revision=attestation["revision"], ) + bound_attestation["runtime_libraries_sha256"] = libraries_sha256 + bound_attestation["runtime_receipt_sha256"] = receipt_sha256 manifest["candidate"] = bound_attestation temp = manifest_path.with_suffix(".tmp") temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") @@ -332,13 +393,34 @@ def main() -> int: parser.add_argument("--expert-cache-slots", type=int, default=REQUIRED_EXPERT_SLOTS) parser.add_argument("--expert-cache-mib", type=int, default=REQUIRED_EXPERT_CACHE_MIB) parser.add_argument("--gpu-layers", type=int, default=99) + parser.add_argument("--signer-principal", required=True) + parser.add_argument("--signing-key", type=Path, required=True) + parser.add_argument("--execution-challenge", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--authorization-issued-unix", type=int, required=True) + parser.add_argument("--authorization-expires-unix", type=int, required=True) parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() try: validate_runtime_config(args) + reject_loader_overrides() + authorization = execution_authorization( + lane=CANDIDATE_LANE, + challenge=args.execution_challenge, + run_id=args.run_id, + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + ) + output = resolved(args.output) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") + validate_signing_identity( + args.signing_key, + args.signer_principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + forbidden_root=output, + ) exporter = resolved(args.exporter) if not exporter.is_file() or not os.access(exporter, os.X_OK): raise PreflightError(f"trace exporter is not executable: {exporter}") @@ -368,7 +450,6 @@ def main() -> int: target_tokens=args.context - args.decode_steps, ) attestation = candidate_attestation(args, exporter, exporter_sha256) - output = resolved(args.output) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") preflight_audit = run_strix_preflight( @@ -420,7 +501,25 @@ def main() -> int: bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) bind_candidate_attestation(output, attestation, accelerator, exporter, exporter_sha256) - bundle = TraceBundle(output) + bind_execution_authorization(output, authorization) + seal_bundle( + output, + private_key=args.signing_key, + principal=args.signer_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + trusted_signers=APPROVED_TRACE_SIGNERS, + ) + bundle = TraceBundle( + output, + verifier=TraceVerifier.production( + args.signer_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + ), + ) if bundle.manifest.get("runtime") != "llama.cpp": raise PreflightError("llama exporter wrote a non-llama.cpp trace") if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index d97eb2983dd7..e9a4c383417d 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -12,13 +12,18 @@ from trace_format import ( ADMITTED_BATCH, ADMITTED_UBATCH, + APPROVED_TRACE_SIGNERS, + CANDIDATE_LANE, CORPUS_SHA256, MODEL_SHA256, REQUIRED_EXPERT_CACHE_MIB, REQUIRED_EXPERT_SLOTS, TraceError, + execution_authorization, + reject_loader_overrides, sha256_file, strict_json_loads, + validate_signing_identity, ) CORPORA = ( @@ -30,7 +35,12 @@ def run(command: list[str]) -> None: - print("exec:", " ".join(command), file=sys.stderr) + displayed = list(command) + if "--signing-key" in displayed: + index = displayed.index("--signing-key") + if index + 1 < len(displayed): + displayed[index + 1] = "" + print("exec:", " ".join(displayed), file=sys.stderr) result = subprocess.run(command, check=False) if result.returncode != 0: raise RuntimeError(f"command failed with status {result.returncode}") @@ -119,6 +129,12 @@ def main() -> int: parser.add_argument("--expert-cache-slots", type=int, default=REQUIRED_EXPERT_SLOTS) parser.add_argument("--expert-cache-mib", type=int, default=REQUIRED_EXPERT_CACHE_MIB) parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) + parser.add_argument("--signer-principal", required=True) + parser.add_argument("--signing-key", type=Path, required=True) + parser.add_argument("--execution-challenge", required=True) + parser.add_argument("--run-id-prefix", required=True) + parser.add_argument("--authorization-issued-unix", type=int, required=True) + parser.add_argument("--authorization-expires-unix", type=int, required=True) args = parser.parse_args() try: @@ -135,12 +151,27 @@ def main() -> int: if args.expert_cache_mib != REQUIRED_EXPERT_CACHE_MIB: raise PreflightError( f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_CACHE_MIB} MiB expert cache") + reject_loader_overrides() + execution_authorization( + lane=CANDIDATE_LANE, + challenge=args.execution_challenge, + run_id=f"{args.run_id_prefix}-preflight", + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + ) + output_candidate = resolved(args.output) if not args.llama_only: raise PreflightError( "cross-runtime capture must run on separate Strix and Apple hosts; " "use --llama-only here and compare completed bundles with trace_format.py") + validate_signing_identity( + args.signing_key, + args.signer_principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + forbidden_root=output_candidate, + ) repo = resolved(args.repo) - output = require_nvme_path(args.output, "matrix output") + output = require_nvme_path(output_candidate, "matrix output") model = require_nvme_path(args.model, "model") if not model.is_file(): raise PreflightError(f"model is not a file: {model}") @@ -222,6 +253,7 @@ def main() -> int: for corpus in corpus_records: stem = Path(corpus["name"]).stem case = f"{stem}-c{context}-ub{ubatch}" + run_id = f"{args.run_id_prefix}-{case}" llama_output = output / "llama" / case prompt = prepared_prompts[corpus["name"]]["path"] provenance = prepared_prompts[corpus["name"]]["provenance_path"] @@ -250,6 +282,12 @@ def main() -> int: "--device", args.device, "--expert-cache-slots", str(args.expert_cache_slots), "--expert-cache-mib", str(args.expert_cache_mib), + "--signer-principal", args.signer_principal, + "--signing-key", str(resolved(args.signing_key)), + "--execution-challenge", args.execution_challenge, + "--run-id", run_id, + "--authorization-issued-unix", str(args.authorization_issued_unix), + "--authorization-expires-unix", str(args.authorization_expires_unix), *common, ]) results.append({ @@ -257,6 +295,7 @@ def main() -> int: "status": "BRINGUP TRACE CAPTURED", "cross_runtime_status": "INCOMPLETE", "trace": str(llama_output), + "run_id": run_id, }) summary = { @@ -268,6 +307,9 @@ def main() -> int: "candidate_revision": args.candidate_revision, "base_revision": args.base_revision, "candidate_diff_sha256": args.candidate_diff_sha256, + "signer_principal": args.signer_principal, + "execution_challenge": args.execution_challenge, + "run_id_prefix": args.run_id_prefix, "corpora": corpus_records, "prompts": prompt_records, "contexts": args.contexts, diff --git a/tools/deepseek-v41-trace/test-install.cmake b/tools/deepseek-v41-trace/test-install.cmake new file mode 100644 index 000000000000..d07ff43b2f93 --- /dev/null +++ b/tools/deepseek-v41-trace/test-install.cmake @@ -0,0 +1,64 @@ +if(NOT DEFINED DSV41_BUILD_DIR OR NOT DEFINED DSV41_INSTALL_ROOT OR + NOT DEFINED DSV41_COMPONENT OR NOT DEFINED DSV41_EXECUTABLE OR + NOT DEFINED DSV41_SOURCE_EXECUTABLE OR + NOT DEFINED DSV41_REVISION OR NOT DEFINED DSV41_PYTHON OR + NOT DEFINED DSV41_RECEIPT OR NOT DEFINED DSV41_VERIFY_SCRIPT) + message(FATAL_ERROR "DeepSeek V4.1 install smoke test arguments are incomplete") +endif() + +file(REMOVE_RECURSE "${DSV41_INSTALL_ROOT}") +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "${DSV41_BUILD_DIR}" + --prefix "${DSV41_INSTALL_ROOT}" + --component "${DSV41_COMPONENT}" + --config "${DSV41_CONFIG}" + RESULT_VARIABLE install_result + OUTPUT_VARIABLE install_output + ERROR_VARIABLE install_error) +if(NOT install_result EQUAL 0) + message(FATAL_ERROR "DeepSeek V4.1 component install failed:\n${install_output}${install_error}") +endif() + +file(SHA256 "${DSV41_SOURCE_EXECUTABLE}" source_executable_sha256) +file(SHA256 "${DSV41_INSTALL_ROOT}/bin/${DSV41_EXECUTABLE}" installed_executable_sha256) +if(NOT source_executable_sha256 STREQUAL installed_executable_sha256) + message(FATAL_ERROR "installed DeepSeek V4.1 trace executable differs from linked bytes") +endif() + +execute_process( + COMMAND "${DSV41_PYTHON}" "${DSV41_VERIFY_SCRIPT}" + --receipt "${DSV41_RECEIPT}" + --install-root "${DSV41_INSTALL_ROOT}" + RESULT_VARIABLE receipt_result + OUTPUT_VARIABLE receipt_output + ERROR_VARIABLE receipt_error) +if(NOT receipt_result EQUAL 0) + message(FATAL_ERROR + "installed DeepSeek V4.1 runtime receipt differs from linked bytes:\n${receipt_output}${receipt_error}") +endif() + +unset(ENV{DYLD_FALLBACK_LIBRARY_PATH}) +unset(ENV{DYLD_FALLBACK_FRAMEWORK_PATH}) +unset(ENV{DYLD_FRAMEWORK_PATH}) +unset(ENV{DYLD_IMAGE_SUFFIX}) +unset(ENV{DYLD_INSERT_LIBRARIES}) +unset(ENV{DYLD_LIBRARY_PATH}) +unset(ENV{DYLD_ROOT_PATH}) +unset(ENV{DYLD_VERSIONED_FRAMEWORK_PATH}) +unset(ENV{DYLD_VERSIONED_LIBRARY_PATH}) +unset(ENV{GGML_BACKEND_PATH}) +unset(ENV{LD_LIBRARY_PATH}) +unset(ENV{LD_PRELOAD}) +unset(ENV{PATH}) + +execute_process( + COMMAND "${DSV41_INSTALL_ROOT}/bin/${DSV41_EXECUTABLE}" --version + RESULT_VARIABLE smoke_result + OUTPUT_VARIABLE smoke_output + ERROR_VARIABLE smoke_error) +if(NOT smoke_result EQUAL 0) + message(FATAL_ERROR "installed DeepSeek V4.1 trace --version failed:\n${smoke_output}${smoke_error}") +endif() +if(NOT smoke_output MATCHES "commit ${DSV41_REVISION}") + message(FATAL_ERROR "installed DeepSeek V4.1 trace reported the wrong revision:\n${smoke_output}") +endif() diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 306a0317dc1a..387e9f0d691f 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -6,16 +6,31 @@ import math import os import re +import stat import struct +import subprocess import sys +import tempfile +import time from dataclasses import dataclass from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Iterable +from typing import Any, Iterable TRACE_FORMAT = "dsv41-trace" TRACE_VERSION = 2 DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" APPROVED_EXPORTERS: dict[str, str] = {} +APPROVED_TRACE_SIGNERS: dict[str, dict[str, str]] = {} +SEAL_FORMAT = "dsv41-trace-bundle-signature" +SEAL_VERSION = 1 +SEAL_NAMESPACE = "dsv41-trace-bundle-v1" +SEAL_DOMAIN_PREFIX = b"dsv41-trace-bundle-v1\n" +SIGNATURE_NAME = "bundle-signature.json" +CANDIDATE_LANE = "strix-llama-candidate-v1" +ORACLE_LANE = "apple-ds4-oracle-v1" +AUTHORIZATION_FORMAT = "dsv41-execution-authorization" +AUTHORIZATION_VERSION = 1 +MAX_AUTHORIZATION_LIFETIME_SECONDS = 24 * 60 * 60 MODEL_SHA256 = "1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42" REPOSITORY = "halo-box/strix-llama.cpp" SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 @@ -53,6 +68,21 @@ MANIFEST_NAME = "manifest.json" EVENTS_NAME = "events.jsonl" BLOBS_DIR = "blobs" +FORBIDDEN_LOADER_ENVIRONMENT = ( + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_IMAGE_SUFFIX", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "DYLD_ROOT_PATH", + "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", + "GGML_BACKEND_PATH", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", +) DTYPE_SIZES = { "f32": 4, @@ -97,6 +127,83 @@ class TraceError(RuntimeError): pass +@dataclass(frozen=True) +class TraceVerifier: + principal: str + trusted_signers: dict[str, dict[str, str]] + ssh_keygen: Path + expected_lane: str + expected_challenge: str + expected_run_id: str + verification_unix: int + seen_run_ids: set[str] | None = None + test_only: bool = False + + @classmethod + def production( + cls, + principal: str, + *, + expected_lane: str, + expected_challenge: str, + expected_run_id: str, + verification_unix: int | None, + seen_run_ids: set[str] | None = None) -> "TraceVerifier": + return cls( + principal=principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + ssh_keygen=trusted_ssh_keygen_path(), + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=int(time.time()) if verification_unix is None else verification_unix, + seen_run_ids=seen_run_ids, + ) + + @classmethod + def for_tests( + cls, + principal: str, + public_key: str, + *, + lane: str, + runtime: str, + runtime_profile: str, + expected_challenge: str, + expected_run_id: str, + verification_unix: int | None = None, + ssh_keygen: Path | None = None, + seen_run_ids: set[str] | None = None) -> "TraceVerifier": + return cls( + principal=principal, + trusted_signers={ + principal: { + "public_key": public_key, + "lane": lane, + "runtime": runtime, + "runtime_profile": runtime_profile, + }, + }, + ssh_keygen=ssh_keygen or trusted_ssh_keygen_path(), + expected_lane=lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=verification_unix, + seen_run_ids=seen_run_ids, + test_only=True, + ) + + +@dataclass(frozen=True) +class BundleFileReceipt: + device: int + inode: int + byte_count: int + modified_ns: int + changed_ns: int + sha256: str + + @dataclass(frozen=True) class Mismatch: classification: str @@ -144,8 +251,15 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def reject_loader_overrides(environment: dict[str, str] | None = None) -> None: + values = os.environ if environment is None else environment + active = sorted(name for name in FORBIDDEN_LOADER_ENVIRONMENT if values.get(name)) + if active: + raise TraceError("production trace execution forbids loader overrides: " + ", ".join(active)) + + def canonical_json(data: Any) -> str: - return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) def strict_json_loads(data: str) -> Any: @@ -157,12 +271,710 @@ def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: result[key] = value return result + def reject_constant(value: str) -> None: + raise TraceError(f"invalid JSON constant: {value}") + try: - return json.loads(data, object_pairs_hook=reject_duplicates) + return json.loads( + data, + object_pairs_hook=reject_duplicates, + parse_constant=reject_constant, + ) except json.JSONDecodeError as error: raise TraceError(f"invalid JSON: {error}") from error +def trusted_ssh_keygen_path() -> Path: + if sys.platform == "win32": + return Path(r"C:\Windows\System32\OpenSSH\ssh-keygen.exe") + if sys.platform in ("darwin", "linux"): + return Path("/usr/bin/ssh-keygen") + raise TraceError(f"unsupported platform for trace signature verification: {sys.platform}") + + +def _validate_ssh_keygen(path: Path) -> Path: + if not path.is_absolute() or path.is_symlink(): + raise TraceError("trusted ssh-keygen path must be an absolute non-symlink") + try: + mode = path.stat().st_mode + except OSError as error: + raise TraceError(f"cannot inspect trusted ssh-keygen: {error}") from error + if not stat.S_ISREG(mode) or not os.access(path, os.X_OK): + raise TraceError("trusted ssh-keygen is not an executable regular file") + try: + result = subprocess.run( + [str(path), "-Y", "verify"], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot probe trusted ssh-keygen: {error}") from error + diagnostic = (result.stdout + result.stderr).lower() + if result.returncode == 0 or any( + marker in diagnostic for marker in ("unknown option", "illegal option", "unknown operation")): + raise TraceError("trusted ssh-keygen lacks required -Y signature support") + return path + + +def _ssh_environment() -> dict[str, str]: + environment = os.environ.copy() + environment.pop("SSH_AUTH_SOCK", None) + environment.pop("SSH_AGENT_PID", None) + return environment + + +def _validate_principal(principal: str) -> None: + if not isinstance(principal, str) or re.fullmatch(r"[A-Za-z0-9._@+-]{1,128}", principal) is None: + raise TraceError("trace signer principal is invalid") + + +def _signer_policy( + trusted_signers: dict[str, dict[str, str]], + principal: str) -> dict[str, str]: + _validate_principal(principal) + policy = trusted_signers.get(principal) + if not isinstance(policy, dict) or set(policy) != { + "public_key", "lane", "runtime", "runtime_profile"}: + raise TraceError(f"trace signer principal is not approved: {principal}") + expected = { + CANDIDATE_LANE: ("llama.cpp", "sibling-lib"), + ORACLE_LANE: ("ds4", "apple-metal"), + }.get(policy.get("lane")) + if expected is None or (policy.get("runtime"), policy.get("runtime_profile")) != expected: + raise TraceError("trace signer policy is invalid") + _normalize_public_key(policy.get("public_key", "")) + return policy + + +def _normalize_public_key(public_key: str, *, allow_comment: bool = False) -> str: + fields = public_key.strip().split() + expected_fields = len(fields) >= 2 if allow_comment else len(fields) == 2 + if not expected_fields or fields[0] != "ssh-ed25519" or re.fullmatch( + r"[A-Za-z0-9+/]+={0,2}", fields[1]) is None: + raise TraceError("trace signer public key must be an OpenSSH Ed25519 key") + return " ".join(fields[:2]) + + +def validate_execution_authorization( + manifest: dict[str, Any], + *, + policy: dict[str, str], + expected_lane: str, + expected_challenge: str, + expected_run_id: str, + verification_unix: int, + seen_run_ids: set[str] | None = None) -> None: + if expected_lane not in {CANDIDATE_LANE, ORACLE_LANE}: + raise TraceError("externally expected execution lane is invalid") + if re.fullmatch(r"[0-9a-f]{64}", expected_challenge) is None: + raise TraceError("externally expected execution challenge is invalid") + run_prefix = "strix-llama-" if expected_lane == CANDIDATE_LANE else "apple-ds4-" + if re.fullmatch(re.escape(run_prefix) + r"[A-Za-z0-9._-]{1,96}", expected_run_id) is None: + raise TraceError("externally expected lane run ID is invalid") + if type(verification_unix) is not int or verification_unix <= 0: + raise TraceError("trace verification time is invalid") + authorization = manifest.get("authorization") + if not isinstance(authorization, dict): + raise TraceError("manifest execution authorization is missing") + _require_exact_keys( + authorization, + {"format", "version", "lane", "challenge", "run_id", "issued_unix", "expires_unix"}, + "manifest execution authorization", + ) + if authorization.get("format") != AUTHORIZATION_FORMAT or ( + authorization.get("version") != AUTHORIZATION_VERSION): + raise TraceError("manifest execution authorization version is invalid") + if authorization.get("lane") != expected_lane or policy["lane"] != expected_lane: + raise TraceError("trace signer is not approved for the expected execution lane") + if authorization.get("challenge") != expected_challenge: + raise TraceError("manifest execution challenge differs from the external challenge") + if authorization.get("run_id") != expected_run_id: + raise TraceError("manifest lane run ID differs from the external run ID") + issued_unix = authorization.get("issued_unix") + expires_unix = authorization.get("expires_unix") + if type(issued_unix) is not int or type(expires_unix) is not int or ( + issued_unix <= 0 or expires_unix <= issued_unix or + expires_unix - issued_unix > MAX_AUTHORIZATION_LIFETIME_SECONDS): + raise TraceError("manifest execution authorization validity window is invalid") + if verification_unix < issued_unix or verification_unix > expires_unix: + raise TraceError("manifest execution authorization is expired or not yet valid") + runtime = manifest.get("runtime") + if runtime != policy["runtime"]: + raise TraceError("trace signer runtime role does not match the signed manifest") + runtime_profile = ( + manifest.get("build", {}).get("runtime_profile", {}).get("name") + if runtime == "llama.cpp" + else manifest.get("accelerator", {}).get("runtime_kind") + ) + if runtime_profile != policy["runtime_profile"]: + raise TraceError("trace signer runtime profile does not match the signed manifest") + if seen_run_ids is not None: + if expected_run_id in seen_run_ids: + raise TraceError("trace lane run ID was reused") + seen_run_ids.add(expected_run_id) + + +def execution_authorization( + *, + lane: str, + challenge: str, + run_id: str, + issued_unix: int, + expires_unix: int) -> dict[str, Any]: + runtime, profile = { + CANDIDATE_LANE: ("llama.cpp", "sibling-lib"), + ORACLE_LANE: ("ds4", "apple-metal"), + }.get(lane, (None, None)) + policy = { + "public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "lane": lane, + "runtime": runtime, + "runtime_profile": profile, + } + authorization = { + "format": AUTHORIZATION_FORMAT, + "version": AUTHORIZATION_VERSION, + "lane": lane, + "challenge": challenge, + "run_id": run_id, + "issued_unix": issued_unix, + "expires_unix": expires_unix, + } + manifest = { + "runtime": runtime, + "authorization": authorization, + "build": {"runtime_profile": {"name": profile}}, + "accelerator": {"runtime_kind": profile}, + } + validate_execution_authorization( + manifest, + policy=policy, + expected_lane=lane, + expected_challenge=challenge, + expected_run_id=run_id, + verification_unix=int(time.time()), + ) + return authorization + + +def bind_execution_authorization(root: Path, authorization: dict[str, Any]) -> None: + manifest_path = root / MANIFEST_NAME + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: + raise TraceError(f"cannot bind execution authorization: {error}") from error + if not isinstance(manifest, dict): + raise TraceError("cannot bind execution authorization to a non-object manifest") + if "authorization" in manifest: + raise TraceError("manifest execution authorization is already present") + manifest["authorization"] = authorization + temporary = manifest_path.with_suffix(".tmp") + temporary.write_bytes(_canonical_json_bytes(manifest)) + os.replace(temporary, manifest_path) + + +def validate_signing_identity( + private_key: Path, + principal: str, + *, + trusted_signers: dict[str, dict[str, str]] = APPROVED_TRACE_SIGNERS, + ssh_keygen: Path | None = None, + forbidden_root: Path | None = None) -> tuple[Path, str]: + policy = _signer_policy(trusted_signers, principal) + expected_key = policy["public_key"] + executable = _validate_ssh_keygen(ssh_keygen or trusted_ssh_keygen_path()) + key_path = private_key.resolve() + if private_key.is_symlink() or not key_path.is_file(): + raise TraceError("trace signing key must be a regular non-symlink file") + key_stat = key_path.stat() + if os.name != "nt": + if key_stat.st_uid != os.getuid(): + raise TraceError("trace signing key is not owned by the current user") + if key_stat.st_mode & 0o077: + raise TraceError("trace signing key permissions are too broad") + if forbidden_root is not None: + try: + key_path.relative_to(forbidden_root.resolve()) + except ValueError: + pass + else: + raise TraceError("trace signing key must be outside the bundle") + try: + result = subprocess.run( + [str(executable), "-y", "-f", str(key_path)], + check=False, + capture_output=True, + text=True, + timeout=10, + env=_ssh_environment(), + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot derive trace signing public key: {error}") from error + if result.returncode != 0: + raise TraceError("cannot derive trace signing public key") + derived_key = _normalize_public_key(result.stdout, allow_comment=True) + if derived_key != _normalize_public_key(expected_key): + raise TraceError("trace signing key does not match the approved signer") + return executable, derived_key + + +def _canonical_json_bytes(data: Any) -> bytes: + return (canonical_json(data) + "\n").encode("ascii") + + +def _bundle_path_parts(relative: str) -> tuple[str, ...]: + if not isinstance(relative, str) or not relative or any(ord(character) > 0x7f for character in relative): + raise TraceError(f"trace path is not portable ASCII: {relative!r}") + if "\\" in relative or "%" in relative or relative.startswith("/") or relative.startswith("//") or ( + re.match(r"^[A-Za-z]:", relative) is not None): + raise TraceError(f"trace path is outside the bundle: {relative}") + parts = relative.split("/") + if any( + not part or part in {".", ".."} or re.fullmatch(r"[A-Za-z0-9._-]+", part) is None + for part in parts): + raise TraceError(f"trace path is not canonical: {relative}") + if PurePosixPath(*parts).as_posix() != relative: + raise TraceError(f"trace path is not canonical: {relative}") + return tuple(parts) + + +def _safe_bundle_file(root: Path, relative: str) -> Path: + parts = _bundle_path_parts(relative) + candidate = root + for part in parts: + candidate = candidate / part + if candidate.is_symlink(): + raise TraceError(f"trace path must not use symlinks: {relative}") + try: + candidate.resolve().relative_to(root) + except ValueError as error: + raise TraceError(f"trace path is outside the bundle: {relative}") from error + if not candidate.is_file(): + raise TraceError(f"trace bundle file is missing or not regular: {relative}") + return candidate + + +def _read_bundle_file( + root: Path, + relative: str, + *, + retain: bool) -> tuple[BundleFileReceipt, bytes | None]: + path = _safe_bundle_file(root, relative) + flags = os.O_RDONLY + if hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as error: + raise TraceError(f"cannot open trace bundle file {relative}: {error}") from error + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise TraceError(f"trace bundle file is not regular: {relative}") + if getattr(before, "st_nlink", 1) != 1: + raise TraceError(f"trace bundle file must not be hard linked: {relative}") + digest = hashlib.sha256() + chunks = [] if retain else None + byte_count = 0 + while True: + chunk = os.read(descriptor, 8 * 1024 * 1024) + if not chunk: + break + digest.update(chunk) + byte_count += len(chunk) + if chunks is not None: + chunks.append(chunk) + after = os.fstat(descriptor) + path_stat = os.stat(path, follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot read trace bundle file {relative}: {error}") from error + finally: + os.close(descriptor) + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + identity_path = ( + path_stat.st_dev, + path_stat.st_ino, + path_stat.st_size, + path_stat.st_mtime_ns, + path_stat.st_ctime_ns, + ) + if identity_before != identity_after or identity_after != identity_path or byte_count != after.st_size: + raise TraceError(f"trace bundle file changed while reading: {relative}") + receipt = BundleFileReceipt( + device=after.st_dev, + inode=after.st_ino, + byte_count=byte_count, + modified_ns=after.st_mtime_ns, + changed_ns=after.st_ctime_ns, + sha256=digest.hexdigest(), + ) + return receipt, b"".join(chunks) if chunks is not None else None + + +def _read_canonical_json( + root: Path, + relative: str) -> tuple[dict[str, Any], bytes, BundleFileReceipt]: + receipt, retained = _read_bundle_file(root, relative, retain=True) + assert retained is not None + try: + data = retained + text = data.decode("ascii") + record = strict_json_loads(text) + except (UnicodeError, TraceError) as error: + raise TraceError(f"cannot read canonical JSON {relative}: {error}") from error + if not isinstance(record, dict) or data != _canonical_json_bytes(record): + raise TraceError(f"trace JSON is not canonical: {relative}") + return record, data, receipt + + +def _read_canonical_jsonl( + root: Path, + relative: str) -> tuple[list[dict[str, Any]], bytes, BundleFileReceipt]: + receipt, retained = _read_bundle_file(root, relative, retain=True) + assert retained is not None + data = retained + if not data or not data.endswith(b"\n"): + raise TraceError(f"trace JSONL is empty or truncated: {relative}") + records = [] + for line_number, raw in enumerate(data.splitlines(keepends=True), 1): + try: + text = raw.decode("ascii") + record = strict_json_loads(text) + except (UnicodeError, TraceError) as error: + raise TraceError(f"invalid JSONL at {relative}:{line_number}: {error}") from error + if not isinstance(record, dict) or raw != _canonical_json_bytes(record): + raise TraceError(f"trace JSONL is not canonical: {relative}:{line_number}") + records.append(record) + return records, data, receipt + + +def _bundle_domain( + root: Path, +) -> tuple[ + bytes, + dict[str, Any], + list[dict[str, Any]], + dict[str, BundleFileReceipt], + dict[str, bytes], +]: + if root.is_symlink(): + raise TraceError("trace root must not be a symlink") + try: + canonical_root = root.resolve(strict=True) + except OSError as error: + raise TraceError(f"cannot resolve trace root: {error}") from error + if not canonical_root.is_dir(): + raise TraceError("trace root is not a directory") + + manifest, manifest_bytes, manifest_receipt = _read_canonical_json(canonical_root, MANIFEST_NAME) + events, events_bytes, events_receipt = _read_canonical_jsonl(canonical_root, EVENTS_NAME) + expected_paths = {MANIFEST_NAME, EVENTS_NAME} + receipts = { + MANIFEST_NAME: manifest_receipt, + EVENTS_NAME: events_receipt, + } + contents = { + MANIFEST_NAME: manifest_bytes, + EVENTS_NAME: events_bytes, + } + + for event in events: + blob = event.get("blob") + if not isinstance(blob, str): + raise TraceError("event blob reference is invalid") + expected_paths.add(blob) + + prompt = manifest.get("prompt") + provenance = prompt.get("provenance") if isinstance(prompt, dict) else None + if not isinstance(provenance, dict) or not isinstance(provenance.get("path"), str): + raise TraceError("prompt provenance reference is missing") + expected_paths.add(provenance["path"]) + + audits = manifest.get("audits") + if not isinstance(audits, dict): + raise TraceError("manifest audit envelope is invalid") + audit_paths = [] + referenced_metadata_paths = {provenance["path"]} + for phase in ("pre", "post"): + phase_audits = audits.get(phase) + if not isinstance(phase_audits, dict): + raise TraceError(f"manifest {phase} audit set is invalid") + for reference in phase_audits.values(): + if not isinstance(reference, dict) or not isinstance(reference.get("path"), str): + raise TraceError(f"manifest {phase} audit reference is invalid") + if reference["path"] in referenced_metadata_paths: + raise TraceError(f"trace metadata path is referenced more than once: {reference['path']}") + referenced_metadata_paths.add(reference["path"]) + audit_paths.append(reference["path"]) + expected_paths.add(reference["path"]) + + for relative in audit_paths: + record, data, receipt = _read_canonical_json(canonical_root, relative) + receipts[relative] = receipt + contents[relative] = data + audit = record.get("data", {}).get("audit") + if isinstance(audit, dict) and isinstance(audit.get("path"), str): + audit_jsonl = audit["path"] + if audit_jsonl in referenced_metadata_paths: + raise TraceError(f"trace metadata path is referenced more than once: {audit_jsonl}") + referenced_metadata_paths.add(audit_jsonl) + expected_paths.add(audit_jsonl) + _records, jsonl_data, jsonl_receipt = _read_canonical_jsonl(canonical_root, audit_jsonl) + receipts[audit_jsonl] = jsonl_receipt + contents[audit_jsonl] = jsonl_data + + _provenance, provenance_data, provenance_receipt = _read_canonical_json( + canonical_root, + provenance["path"], + ) + receipts[provenance["path"]] = provenance_receipt + contents[provenance["path"]] = provenance_data + + actual_paths = set() + try: + for path in canonical_root.rglob("*"): + relative = path.relative_to(canonical_root).as_posix() + if path.is_symlink(): + raise TraceError(f"trace bundle contains a symlink: {relative}") + if path.is_file(): + if relative != SIGNATURE_NAME: + actual_paths.add(relative) + elif not path.is_dir(): + raise TraceError(f"trace bundle contains a nonregular entry: {relative}") + except OSError as error: + raise TraceError(f"cannot enumerate trace bundle: {error}") from error + if actual_paths != expected_paths: + missing = sorted(expected_paths - actual_paths) + extra = sorted(actual_paths - expected_paths) + detail = [] + if missing: + detail.append("missing " + ", ".join(missing)) + if extra: + detail.append("unexpected " + ", ".join(extra)) + raise TraceError(f"trace signed file set is invalid: {'; '.join(detail)}") + + records = [] + for relative in sorted(expected_paths, key=lambda value: value.encode("ascii")): + _bundle_path_parts(relative) + data = contents.get(relative) + if data is not None: + byte_count = len(data) + digest = sha256_bytes(data) + else: + receipt, _retained = _read_bundle_file(canonical_root, relative, retain=False) + receipts[relative] = receipt + byte_count = receipt.byte_count + digest = receipt.sha256 + records.append({ + "path": relative, + "byte_count": byte_count, + "sha256": digest, + }) + domain = SEAL_DOMAIN_PREFIX + _canonical_json_bytes(records) + return domain, manifest, events, receipts, contents + + +def seal_bundle( + root: Path, + *, + private_key: Path, + principal: str, + expected_lane: str, + expected_challenge: str, + expected_run_id: str, + verification_unix: int | None = None, + trusted_signers: dict[str, dict[str, str]] = APPROVED_TRACE_SIGNERS, + ssh_keygen: Path | None = None) -> str: + root = root.resolve() + executable, _public_key = validate_signing_identity( + private_key, + principal, + trusted_signers=trusted_signers, + ssh_keygen=ssh_keygen, + forbidden_root=root, + ) + key_path = private_key.resolve() + signature_path = root / SIGNATURE_NAME + if signature_path.exists() or signature_path.is_symlink(): + raise TraceError("trace signature envelope already exists") + domain, manifest, _events, _receipts, _contents = _bundle_domain(root) + validate_execution_authorization( + manifest, + policy=_signer_policy(trusted_signers, principal), + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=int(time.time()) if verification_unix is None else verification_unix, + ) + try: + with tempfile.TemporaryDirectory(prefix="dsv41-trace-sign-") as signing_temp: + signing_root = Path(signing_temp).resolve() + if os.name != "nt": + signing_root.chmod(0o700) + domain_path = signing_root / "bundle-domain" + signature_file = signing_root / "bundle-domain.sig" + with domain_path.open("xb") as stream: + stream.write(domain) + stream.flush() + os.fsync(stream.fileno()) + if os.name != "nt": + domain_path.chmod(0o600) + if signature_file.exists() or signature_file.is_symlink(): + raise TraceError("temporary trace signature output already exists") + result = subprocess.run( + [ + str(executable), + "-Y", "sign", + "-f", str(key_path), + "-n", SEAL_NAMESPACE, + str(domain_path), + ], + stdin=subprocess.DEVNULL, + check=False, + capture_output=True, + timeout=30, + env=_ssh_environment(), + ) + if result.returncode != 0: + raise TraceError("trusted ssh-keygen failed to sign trace bundle") + signature_receipt, signature_bytes = _read_bundle_file( + signing_root, + signature_file.name, + retain=True, + ) + if signature_receipt.byte_count == 0 or signature_receipt.sha256 != sha256_bytes(signature_bytes or b""): + raise TraceError("trusted ssh-keygen did not create a stable signature") + assert signature_bytes is not None + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot sign trace bundle: {error}") from error + try: + signature = signature_bytes.decode("ascii") + except UnicodeError as error: + raise TraceError("trace signature is not ASCII") from error + if not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or not signature.endswith( + "-----END SSH SIGNATURE-----\n"): + raise TraceError("trusted ssh-keygen returned an invalid signature") + envelope = { + "format": SEAL_FORMAT, + "version": SEAL_VERSION, + "namespace": SEAL_NAMESPACE, + "principal": principal, + "domain_sha256": sha256_bytes(domain), + "signature": signature, + } + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=root, + prefix=".bundle-signature.", + delete=False) as stream: + temporary_path = Path(stream.name) + stream.write(_canonical_json_bytes(envelope)) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary_path, signature_path) + except FileExistsError as error: + raise TraceError("trace signature envelope already exists") from error + except OSError as error: + raise TraceError(f"cannot install trace signature envelope: {error}") from error + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + return envelope["domain_sha256"] + + +def verify_bundle_seal( + root: Path, + verifier: TraceVerifier, +) -> tuple[ + dict[str, Any], + list[dict[str, Any]], + str, + dict[str, BundleFileReceipt], + dict[str, bytes], +]: + policy = _signer_policy(verifier.trusted_signers, verifier.principal) + approved_key = _normalize_public_key(policy["public_key"]) + executable = _validate_ssh_keygen(verifier.ssh_keygen) + root = root.resolve() + envelope, _envelope_bytes, _envelope_receipt = _read_canonical_json(root, SIGNATURE_NAME) + _require_exact_keys( + envelope, + {"format", "version", "namespace", "principal", "domain_sha256", "signature"}, + "trace signature envelope", + ) + if envelope.get("format") != SEAL_FORMAT or envelope.get("version") != SEAL_VERSION or ( + envelope.get("namespace") != SEAL_NAMESPACE): + raise TraceError("trace signature envelope version is invalid") + if envelope.get("principal") != verifier.principal: + raise TraceError("trace signature principal differs from the externally expected signer") + signature = envelope.get("signature") + if not isinstance(signature, str) or not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or ( + not signature.endswith("-----END SSH SIGNATURE-----\n")): + raise TraceError("trace signature envelope contains an invalid signature") + domain, manifest, events, receipts, contents = _bundle_domain(root) + validate_execution_authorization( + manifest, + policy=policy, + expected_lane=verifier.expected_lane, + expected_challenge=verifier.expected_challenge, + expected_run_id=verifier.expected_run_id, + verification_unix=verifier.verification_unix, + seen_run_ids=verifier.seen_run_ids, + ) + domain_sha256 = sha256_bytes(domain) + if envelope.get("domain_sha256") != domain_sha256: + raise TraceError("trace signed-domain SHA-256 mismatch") + with tempfile.TemporaryDirectory(prefix="dsv41-trace-verify-") as temp: + temporary = Path(temp) + allowed_signers = temporary / "allowed_signers" + signature_file = temporary / "signature" + allowed_signers.write_text( + f"{verifier.principal} {approved_key}\n", + encoding="ascii", + ) + signature_file.write_text(signature, encoding="ascii") + try: + result = subprocess.run( + [ + str(executable), + "-Y", "verify", + "-f", str(allowed_signers), + "-I", verifier.principal, + "-n", SEAL_NAMESPACE, + "-s", str(signature_file), + ], + input=domain, + check=False, + capture_output=True, + timeout=30, + env=_ssh_environment(), + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot verify trace bundle signature: {error}") from error + if result.returncode != 0: + raise TraceError("trace bundle signature verification failed") + return manifest, events, domain_sha256, receipts, contents + + def _require_exact_keys(record: dict[str, Any], keys: set[str], label: str) -> None: missing = sorted(keys - set(record)) extra = sorted(set(record) - keys) @@ -734,46 +1546,69 @@ def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: class TraceBundle: - def __init__(self, root: Path, verify_blobs: bool = True): + def __init__( + self, + root: Path, + verify_blobs: bool = True, + *, + verifier: TraceVerifier | None = None, + signer_principal: str | None = None, + expected_lane: str | None = None, + expected_challenge: str | None = None, + expected_run_id: str | None = None, + verification_unix: int | None = None, + seen_run_ids: set[str] | None = None): if root.is_symlink(): raise TraceError("trace root must not be a symlink") self.root = root.resolve() - try: - self.manifest = strict_json_loads(self._path(MANIFEST_NAME).read_text(encoding="ascii")) - except (OSError, UnicodeError, TraceError) as error: - raise TraceError(f"cannot read manifest: {error}") from error + if verifier is None: + if None in (signer_principal, expected_lane, expected_challenge, expected_run_id): + raise TraceError( + "external signer, lane, challenge, and run ID expectations are required") + verifier = TraceVerifier.production( + signer_principal, + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=verification_unix, + seen_run_ids=seen_run_ids, + ) + elif any(value is not None for value in ( + signer_principal, expected_lane, expected_challenge, expected_run_id, + verification_unix, seen_run_ids)): + raise TraceError("trace verifier cannot be combined with separate verification inputs") + self.signer_principal = verifier.principal + ( + self.manifest, + sealed_events, + self.seal_sha256, + self._file_receipts, + self._retained_files, + ) = verify_bundle_seal(self.root, verifier) if self.manifest.get("trace_format") != TRACE_FORMAT: raise TraceError("manifest trace_format mismatch") if self.manifest.get("trace_version") != TRACE_VERSION: raise TraceError("manifest trace_version mismatch") self._validate_manifest() - self.events = self._read_events(verify_blobs) + self.events = self._validate_sealed_events(sealed_events, verify_blobs) if self.manifest.get("event_count") != len(self.events): raise TraceError("manifest event_count mismatch") self._validate_coverage() - def _read_events(self, verify_blobs: bool) -> list[dict[str, Any]]: + def _validate_sealed_events( + self, + sealed_events: list[dict[str, Any]], + verify_blobs: bool) -> list[dict[str, Any]]: result = [] - try: - stream: BinaryIO - with self._path(EVENTS_NAME).open("rb") as stream: - for line_number, raw in enumerate(stream, 1): - if not raw.endswith(b"\n"): - raise TraceError(f"events.jsonl is truncated at line {line_number}") - try: - event = strict_json_loads(raw.decode("ascii")) - except (UnicodeError, TraceError) as error: - raise TraceError(f"invalid event at line {line_number}: {error}") from error - validate_event(event) - if verify_blobs: - data = self.read_blob(event) - if len(data) != event["byte_count"]: - raise TraceError(f"truncated blob for event line {line_number}") - if sha256_bytes(data) != event["sha256"]: - raise TraceError(f"corrupt blob for event line {line_number}") - result.append(event) - except OSError as error: - raise TraceError(f"cannot read events: {error}") from error + for line_number, event in enumerate(sealed_events, 1): + validate_event(event) + if verify_blobs: + data = self.read_blob(event) + if len(data) != event["byte_count"]: + raise TraceError(f"truncated blob for event line {line_number}") + if sha256_bytes(data) != event["sha256"]: + raise TraceError(f"corrupt blob for event line {line_number}") + result.append(event) return result def _validate_manifest(self) -> None: @@ -796,6 +1631,7 @@ def _validate_manifest(self) -> None: "environment", "paths", "storage_policy", + "authorization", "audits", "expected", } @@ -811,7 +1647,11 @@ def _validate_manifest(self) -> None: if not isinstance(self.manifest["build"], dict): raise TraceError("manifest build is invalid") build_keys = ( - {"number", "info", "compiler", "target", "path", "sha256", "runtime_libraries"} + { + "number", "info", "compiler", "target", "path", "sha256", + "runtime_profile", "runtime_receipt_sha256", + "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor", + } if self.manifest["runtime"] == "llama.cpp" else {"compiler", "target", "path", "sha256"} ) @@ -822,7 +1662,9 @@ def _validate_manifest(self) -> None: if self.manifest["runtime"] == "ds4" and ( APPROVED_EXPORTERS.get(build_sha256) != DS4_REVISION): raise TraceError("ds4 exporter is not approved for the pinned ds4 revision") - for key in build_keys - {"sha256", "number", "runtime_libraries"}: + for key in build_keys - { + "sha256", "number", "runtime_profile", "runtime_receipt_sha256", + "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor"}: value = self.manifest["build"].get(key) if not isinstance(value, str) or not value: raise TraceError(f"manifest build {key} is invalid") @@ -836,35 +1678,81 @@ def _validate_manifest(self) -> None: libraries = self.manifest["build"]["runtime_libraries"] if not isinstance(libraries, list) or not libraries: raise TraceError("manifest runtime library identities are invalid") + post_libraries = self.manifest["build"]["runtime_libraries_post"] + if post_libraries != libraries: + raise TraceError("manifest runtime library closure changed during trace generation") + module_monitor = self.manifest["build"]["runtime_module_monitor"] + if not isinstance(module_monitor, dict): + raise TraceError("manifest runtime module monitor is invalid") + _require_exact_keys( + module_monitor, + {"mechanism", "checked_after_trace", "project_additions"}, + "manifest runtime module monitor", + ) + if module_monitor["mechanism"] not in {"dyld-add-image", "pre-post-snapshot"}: + raise TraceError("manifest runtime module monitor mechanism is invalid") + if module_monitor["checked_after_trace"] is not True: + raise TraceError("manifest runtime module monitor did not complete") + if module_monitor["project_additions"] != []: + raise TraceError("manifest records a runtime module addition during trace generation") + profile = self.manifest["build"]["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("manifest runtime profile is invalid") + _require_exact_keys( + profile, + {"name", "components", "selected_backend_component"}, + "manifest runtime profile", + ) + profile_name = profile.get("name") + components = profile.get("components") + selected_backend_component = profile.get("selected_backend_component") + if profile_name not in {"co-located", "sibling-lib"}: + raise TraceError("manifest runtime profile name is invalid") + if not isinstance(components, list) or components != sorted(components) or ( + len(components) != len(set(components))) or any( + not isinstance(component, str) or re.fullmatch(r"[a-z0-9-]+", component) is None + for component in components): + raise TraceError("manifest runtime profile components are invalid") + if not {"llama-common", "llama", "ggml", "ggml-base"}.issubset(set(components)): + raise TraceError("manifest runtime profile is missing core components") + if not isinstance(selected_backend_component, str) or ( + selected_backend_component not in components or + selected_backend_component in {"ggml", "ggml-base"} or + not selected_backend_component.startswith("ggml-")): + raise TraceError("manifest selected backend component is invalid") roles = set() paths = set() + library_components = set() previous_path = None - expected_roles = {"build-info", "llama", "ggml", "selected-backend"} executable_path = PurePosixPath(build_path) binary_directory = executable_path.parent library_directory = binary_directory.parent / "lib" for library in libraries: _require_exact_keys( library, - {"path", "sha256", "roles", "revision"}, + {"component", "filename", "path", "sha256", "role", "revision"}, "manifest runtime library", ) + component = library.get("component") + filename = library.get("filename") path = library.get("path") digest = library.get("sha256") - library_roles = library.get("roles") + role = library.get("role") revision = library.get("revision") - if not isinstance(library_roles, list) or any( - not isinstance(role, str) or role not in expected_roles - for role in library_roles): - raise TraceError("manifest runtime library roles are invalid") - if len(library_roles) != len(set(library_roles)): - raise TraceError("manifest runtime library role is duplicated") - if library_roles != sorted(library_roles): - raise TraceError("manifest runtime library roles are not sorted") - for role in library_roles: - if role in roles: - raise TraceError("manifest runtime library role is duplicated") - roles.add(role) + if component not in components or component in library_components: + raise TraceError("manifest runtime library component is invalid") + library_components.add(component) + if not isinstance(filename, str) or PurePosixPath(filename).name != filename: + raise TraceError("manifest runtime library filename is invalid") + expected_role = { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + selected_backend_component: "selected-backend", + }.get(component, f"runtime:{component}") + if role != expected_role or role in roles: + raise TraceError("manifest runtime library role is invalid") + roles.add(role) if not isinstance(path, str) or not path.startswith("/") or ".." in PurePosixPath(path).parts or ( str(PurePosixPath(path)) != path): raise TraceError("manifest runtime library path is not canonical") @@ -885,13 +1773,38 @@ def _validate_manifest(self) -> None: previous_path = path if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: raise TraceError("manifest runtime library SHA-256 is invalid") - if set(library_roles) & {"build-info", "ggml"}: + if component in {"llama-common", "ggml-base"}: if revision != self.manifest["revision"]: raise TraceError("manifest runtime library revision is invalid") elif revision is not None: raise TraceError("manifest runtime library revision is unexpected") - if roles != expected_roles: - raise TraceError("manifest runtime library identities are incomplete") + expected_directory = binary_directory if profile_name == "co-located" else library_directory + if runtime_path.parent != expected_directory or runtime_path.name != filename: + raise TraceError("manifest runtime library path differs from the runtime profile") + if library_components != set(components): + raise TraceError("manifest runtime library set differs from the runtime profile") + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": self.manifest["revision"], + "profile": profile_name, + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in libraries + ], + key=lambda item: item["component"], + ), + } + receipt_sha256 = self.manifest["build"].get("runtime_receipt_sha256") + if not isinstance(receipt_sha256, str) or receipt_sha256 != sha256_bytes( + canonical_json(receipt).encode("ascii")): + raise TraceError("manifest runtime receipt SHA-256 is invalid") for section in ("model", "prompt"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") @@ -987,7 +1900,7 @@ def _validate_manifest(self) -> None: if provenance.get("path") != f"provenance/{provenance_sha256}.json": raise TraceError("prompt provenance path is not content addressed") try: - provenance_bytes = self._path(provenance["path"]).read_bytes() + provenance_bytes = self._read_verified_file(provenance["path"]) provenance_record = strict_json_loads(provenance_bytes.decode("ascii")) except (OSError, UnicodeError, TraceError) as error: raise TraceError(f"cannot read prompt provenance: {error}") from error @@ -1031,6 +1944,7 @@ def _validate_manifest(self) -> None: "executable_path", "executable_sha256", "runtime_libraries_sha256", + "runtime_receipt_sha256", }, "llama.cpp candidate attestation", ) @@ -1041,7 +1955,8 @@ def _validate_manifest(self) -> None: "base_revision", "diff_sha256", "executable_sha256", - "runtime_libraries_sha256"): + "runtime_libraries_sha256", + "runtime_receipt_sha256"): value = candidate.get(key, "") if not isinstance(value, str) or re.fullmatch( r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: @@ -1058,9 +1973,14 @@ def _validate_manifest(self) -> None: if candidate["executable_sha256"] != self.manifest["build"]["sha256"]: raise TraceError("candidate executable SHA-256 does not match the trace build") runtime_libraries_sha256 = sha256_bytes( - canonical_json(self.manifest["build"]["runtime_libraries"]).encode("ascii")) + canonical_json({ + "pre": self.manifest["build"]["runtime_libraries"], + "post": self.manifest["build"]["runtime_libraries_post"], + }).encode("ascii")) if candidate["runtime_libraries_sha256"] != runtime_libraries_sha256: raise TraceError("candidate runtime library identities do not match the trace build") + if candidate["runtime_receipt_sha256"] != self.manifest["build"]["runtime_receipt_sha256"]: + raise TraceError("candidate runtime receipt does not match the trace build") expected_config = { "layer_count": 40, "vocab_size": 129280, @@ -1180,10 +2100,9 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: expected_path = f"audits/{phase}/{digest}.json" if audit_path != expected_path: raise TraceError(f"manifest {phase} {kind} audit path is not content addressed") - evidence_path = self._path(audit_path) try: - evidence = evidence_path.read_bytes() - except OSError as error: + evidence = self._read_verified_file(audit_path) + except TraceError as error: raise TraceError(f"cannot read {phase} {kind} audit evidence: {error}") from error if sha256_bytes(evidence) != digest: raise TraceError(f"{phase} {kind} audit evidence SHA-256 mismatch") @@ -1439,10 +2358,9 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} watchdog JSONL path is invalid") if type(audit_jsonl.get("event_count")) is not int or audit_jsonl["event_count"] < 2: raise TraceError(f"{phase} watchdog JSONL event count is invalid") - jsonl_path = self._path(audit_jsonl["path"]) try: - jsonl_bytes = jsonl_path.read_bytes() - except OSError as error: + jsonl_bytes = self._read_verified_file(audit_jsonl["path"]) + except TraceError as error: raise TraceError(f"cannot read {phase} watchdog JSONL audit: {error}") from error if sha256_bytes(jsonl_bytes) != jsonl_digest: raise TraceError(f"{phase} watchdog JSONL SHA-256 mismatch") @@ -1458,17 +2376,25 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} watchdog JSONL lacks startup evidence") def read_blob(self, event: dict[str, Any]) -> bytes: - try: - return self._path(event["blob"]).read_bytes() - except OSError as error: - raise TraceError(f"cannot read blob {event['blob']}: {error}") from error + return self._read_verified_file(event["blob"]) + + def _read_verified_file(self, relative: str) -> bytes: + retained = self._retained_files.get(relative) + if retained is not None: + return retained + expected = self._file_receipts.get(relative) + if expected is None: + raise TraceError(f"trace file is outside the signed domain: {relative}") + receipt, data = _read_bundle_file(self.root, relative, retain=True) + if receipt != expected: + raise TraceError(f"trace bundle file changed after signature verification: {relative}") + assert data is not None + return data def _path(self, relative: str) -> Path: - relative_path = Path(relative) - if relative_path.is_absolute() or ".." in relative_path.parts: - raise TraceError(f"trace path is outside the bundle: {relative}") + parts = _bundle_path_parts(relative) candidate = self.root - for part in relative_path.parts: + for part in parts: candidate = candidate / part if candidate.is_symlink(): raise TraceError(f"trace path must not use symlinks: {relative}") @@ -1852,6 +2778,10 @@ def report( "left_runtime": left.manifest.get("runtime"), "right_runtime": right.manifest.get("runtime"), "events_compared": len(left.events), + "left_seal_sha256": left.seal_sha256, + "right_seal_sha256": right.seal_sha256, + "left_signer_principal": left.signer_principal, + "right_signer_principal": right.signer_principal, "first_divergence": None, } return { @@ -1860,12 +2790,22 @@ def report( "left_runtime": left.manifest.get("runtime"), "right_runtime": right.manifest.get("runtime"), "events_compared": 0, + "left_seal_sha256": left.seal_sha256, + "right_seal_sha256": right.seal_sha256, + "left_signer_principal": left.signer_principal, + "right_signer_principal": right.signer_principal, "first_divergence": mismatch.as_dict(), } def command_validate(args: argparse.Namespace) -> int: - bundle = TraceBundle(args.bundle) + bundle = TraceBundle( + args.bundle, + signer_principal=getattr(args, "signer_principal", None), + expected_lane=getattr(args, "lane", None), + expected_challenge=getattr(args, "execution_challenge", None), + expected_run_id=getattr(args, "run_id", None), + ) print(canonical_json({ "status": "valid", "runtime": bundle.manifest.get("runtime"), @@ -1875,7 +2815,30 @@ def command_validate(args: argparse.Namespace) -> int: def command_compare(args: argparse.Namespace) -> int: - result = report(TraceBundle(args.left), TraceBundle(args.right)) + left_principal = getattr(args, "left_signer_principal", None) + right_principal = getattr(args, "right_signer_principal", None) + challenge = getattr(args, "execution_challenge", None) + left_run_id = getattr(args, "left_run_id", None) + right_run_id = getattr(args, "right_run_id", None) + seen_run_ids: set[str] = set() + result = report( + TraceBundle( + args.left, + signer_principal=left_principal, + expected_lane=ORACLE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + seen_run_ids=seen_run_ids, + ), + TraceBundle( + args.right, + signer_principal=right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + seen_run_ids=seen_run_ids, + ), + ) text = canonical_json(result) + "\n" if args.report: args.report.write_text(text, encoding="ascii") @@ -1930,7 +2893,31 @@ def local_report(left: TraceBundle, right: TraceBundle, mode: str) -> dict[str, def command_compare_local(args: argparse.Namespace) -> int: - result = local_report(TraceBundle(args.left), TraceBundle(args.right), args.mode) + left_principal = getattr(args, "left_signer_principal", None) + right_principal = getattr(args, "right_signer_principal", None) + challenge = getattr(args, "execution_challenge", None) + left_run_id = getattr(args, "left_run_id", None) + right_run_id = getattr(args, "right_run_id", None) + seen_run_ids: set[str] = set() + result = local_report( + TraceBundle( + args.left, + signer_principal=left_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + seen_run_ids=seen_run_ids, + ), + TraceBundle( + args.right, + signer_principal=right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + seen_run_ids=seen_run_ids, + ), + args.mode, + ) text = canonical_json(result) + "\n" if args.report: args.report.write_text(text, encoding="ascii") @@ -1943,16 +2930,30 @@ def build_parser() -> argparse.ArgumentParser: subparsers = parser.add_subparsers(dest="command", required=True) validate_parser = subparsers.add_parser("validate") validate_parser.add_argument("bundle", type=Path) + validate_parser.add_argument("--signer-principal", required=True) + validate_parser.add_argument("--lane", choices=(CANDIDATE_LANE, ORACLE_LANE), required=True) + validate_parser.add_argument("--execution-challenge", required=True) + validate_parser.add_argument("--run-id", required=True) validate_parser.set_defaults(func=command_validate) compare_parser = subparsers.add_parser("compare") compare_parser.add_argument("left", type=Path) compare_parser.add_argument("right", type=Path) + compare_parser.add_argument("--left-signer-principal", required=True) + compare_parser.add_argument("--right-signer-principal", required=True) + compare_parser.add_argument("--execution-challenge", required=True) + compare_parser.add_argument("--left-run-id", required=True) + compare_parser.add_argument("--right-run-id", required=True) compare_parser.add_argument("--report", type=Path) compare_parser.set_defaults(func=command_compare) local_parser = subparsers.add_parser("compare-local") local_parser.add_argument("mode", choices=("self-consistency", "base-regression")) local_parser.add_argument("left", type=Path) local_parser.add_argument("right", type=Path) + local_parser.add_argument("--left-signer-principal", required=True) + local_parser.add_argument("--right-signer-principal", required=True) + local_parser.add_argument("--execution-challenge", required=True) + local_parser.add_argument("--left-run-id", required=True) + local_parser.add_argument("--right-run-id", required=True) local_parser.add_argument("--report", type=Path) local_parser.set_defaults(func=command_compare_local) return parser diff --git a/tools/deepseek-v41-trace/verify-runtime-install.py b/tools/deepseek-v41-trace/verify-runtime-install.py new file mode 100644 index 000000000000..b305ab1b7e5d --- /dev/null +++ b/tools/deepseek-v41-trace/verify-runtime-install.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +def strict_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--receipt", required=True, type=Path) + parser.add_argument("--install-root", required=True, type=Path) + args = parser.parse_args() + + receipt_bytes = args.receipt.read_bytes() + try: + receipt = json.loads(receipt_bytes, object_pairs_hook=strict_object) + except (UnicodeError, json.JSONDecodeError, ValueError) as error: + parser.error(f"runtime receipt JSON is invalid: {error}") + canonical = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + if receipt_bytes != canonical: + parser.error("runtime receipt JSON is not canonical") + if set(receipt) != {"format", "version", "revision", "profile", "components"} or ( + receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or + re.fullmatch(r"[0-9a-f]{40}", receipt["revision"]) is None or + receipt["profile"] not in {"co-located", "sibling-lib"}): + parser.error("runtime receipt is invalid") + directory = args.install_root / ("bin" if receipt["profile"] == "co-located" else "lib") + components = receipt["components"] + if not isinstance(components, list) or not components: + parser.error("runtime receipt component list is invalid") + if components != sorted(components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + parser.error("runtime receipt component list is not canonical") + names = set() + filenames = set() + digests = set() + for component in components: + if not isinstance(component, dict) or set(component) != { + "component", "filename", "sha256", "revision"}: + parser.error("runtime receipt component is invalid") + filename = component["filename"] + name = component["component"] + digest = component["sha256"] + revision = component["revision"] + if not isinstance(name, str) or re.fullmatch(r"[a-z0-9-]+", name) is None or name in names: + parser.error("runtime receipt component name is invalid") + if not isinstance(filename, str) or Path(filename).name != filename or ( + re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None) or filename in filenames: + parser.error("runtime receipt filename is invalid") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None or digest in digests: + parser.error("runtime receipt component SHA-256 is invalid") + revision_bearing = name in {"llama-common", "ggml-base"} + if (revision_bearing and revision != receipt["revision"]) or ( + not revision_bearing and revision is not None): + parser.error("runtime receipt component revision is invalid") + names.add(name) + filenames.add(filename) + digests.add(digest) + installed = directory / filename + if installed.is_symlink() or not installed.is_file(): + parser.error(f"installed runtime component is missing or not regular: {filename}") + if sha256_file(installed) != digest: + parser.error(f"installed runtime component SHA-256 mismatch: {name}") + if not {"llama-common", "llama", "ggml", "ggml-base"}.issubset(names): + parser.error("runtime receipt is missing core components") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 82386e3f150942c4438aad3b144623d2dcd59851 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 11:43:36 -0700 Subject: [PATCH 35/56] tools: require external trace executable approvals Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 695 ++++++++++++++-- tools/deepseek-v41-trace/README.md | 36 +- tools/deepseek-v41-trace/llama-trace.cpp | 14 + tools/deepseek-v41-trace/preflight.py | 33 +- tools/deepseek-v41-trace/run_ds4.py | 51 +- tools/deepseek-v41-trace/run_llama.py | 257 +++++- tools/deepseek-v41-trace/run_matrix.py | 165 +++- tools/deepseek-v41-trace/trace_format.py | 996 +++++++++++++++++++++-- 8 files changed, 2103 insertions(+), 144 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 3164520fca94..979d28bc5477 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -41,6 +41,8 @@ "llama.cpp": "strix-llama-test-run", "ds4": "apple-ds4-test-run", } +TEST_CANDIDATE_EXPORTER_POLICY_ID = "test-candidate-exporter" +TEST_PROMPT_BUILDER_POLICY_ID = "test-prompt-builder" WATCHDOG_EVENTS = [ { @@ -363,24 +365,115 @@ def replace_watchdog_events(root: Path, phase: str, events: list[dict[str, objec replace_audit_record(root, phase, "watchdog", record) -def provenance_bytes(prompt: bytes = b"abc") -> bytes: +def fixture_prompt_builder_policy( + prompt: bytes, + *, + context: int = 3, + decode_steps: int = 1, + builder_path: str = "/home/repo/build/bin/llama-deepseek-v41-prompt-builder", + builder_sha256: str = "8" * 64, + source_root: str = "/home/repo") -> dict[str, object]: + runtime_components = [ + ("ggml", "libggml.so", "4", None), + ("ggml-base", "libggml-base.so", "5", "a" * 40), + ("ggml-hip", "libggml-hip.so", "6", None), + ("llama", "libllama.so", "7", None), + ("llama-common", "libllama-common.so", "9", "a" * 40), + ] + runtime_profile = { + "name": "sibling-lib", + "components": sorted(component for component, *_rest in runtime_components), + "selected_backend_component": "ggml-hip", + } + return { + "runtime": "llama.cpp", + "runtime_profile": runtime_profile, + "repository": trace.REPOSITORY, + "revision": "a" * 40, + "install_root": str(Path(builder_path).parent.parent), + "executable_path": builder_path, + "executable_sha256": builder_sha256, + "source_root": source_root, + "runtime_receipt": { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "a" * 40, + "profile": "sibling-lib", + "components": [ + { + "component": component, + "filename": filename, + "sha256": digest * 64, + "revision": revision, + } + for component, filename, digest, revision in runtime_components + ], + }, + "model_sha256": trace.MODEL_SHA256, + "corpora": dict(trace.CORPUS_SHA256), + "prompts": [{ + "corpus_name": "correctness-prose.txt", + "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "context": context, + "decode_steps": decode_steps, + "target_tokens": context - decode_steps, + "prompt_sha256": trace.sha256_bytes(prompt), + "prompt_byte_count": len(prompt), + "add_bos": True, + }], + } + + +def materialize_policy_runtime(policy: dict[str, object]) -> None: + library_root = Path(policy["install_root"]) / "lib" + library_root.mkdir(parents=True, exist_ok=True) + for component in policy["runtime_receipt"]["components"]: + path = library_root / component["filename"] + path.write_bytes(component["component"].encode("ascii")) + component["sha256"] = trace.sha256_file(path) + + +def provenance_bytes( + prompt: bytes = b"abc", + *, + context: int = 3, + decode_steps: int = 1) -> bytes: + policy = fixture_prompt_builder_policy(prompt, context=context, decode_steps=decode_steps) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) record = { "format": "dsv41-prompt-provenance", "version": 1, "corpus_name": "correctness-prose.txt", "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "corpus_path": "/home/repo/tests/corpus/correctness-prose.txt", "model_sha256": trace.MODEL_SHA256, "prompt_sha256": trace.sha256_bytes(prompt), "prompt_byte_count": len(prompt), - "target_tokens": 2, - "actual_tokens": 2, + "context": context, + "decode_steps": decode_steps, + "target_tokens": context - decode_steps, + "actual_tokens": context - decode_steps, + "builder_approval_id": TEST_PROMPT_BUILDER_POLICY_ID, + "builder_approval_sha256": policy_sha256, + "builder_path": policy["executable_path"], "builder_sha256": "8" * 64, + "builder_revision": "a" * 40, + "builder_runtime_profile": policy["runtime_profile"], } return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") -def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: - provenance_sha256 = trace.sha256_bytes(provenance_bytes(prompt)) +def manifest( + runtime: str = "llama.cpp", + prompt: bytes = b"abc", + *, + context: int = 3, + decode_steps: int = 1) -> dict: + provenance_sha256 = trace.sha256_bytes( + provenance_bytes(prompt, context=context, decode_steps=decode_steps)) is_ds4 = runtime == "ds4" storage = DS4_STORAGE_ATTESTATION if is_ds4 else STORAGE_ATTESTATION audit_kinds = ("memory", "swap", "runner") if is_ds4 else ("memory", "swap", "watchdog") @@ -420,13 +513,6 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: result = { "runtime": runtime, "revision": trace.DS4_REVISION if is_ds4 else "a" * 40, - "authorization": trace.execution_authorization( - lane=trace.ORACLE_LANE if is_ds4 else trace.CANDIDATE_LANE, - challenge=TEST_CHALLENGE, - run_id=TEST_RUN_IDS[runtime], - issued_unix=TEST_AUTH_ISSUED, - expires_unix=TEST_AUTH_EXPIRES, - ), "build": ( { "compiler": "clang", @@ -472,15 +558,15 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "byte_count": len(prompt), "corpus_name": "correctness-prose.txt", "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], - "target_tokens": 2, + "target_tokens": context - decode_steps, "provenance": { "path": f"provenance/{provenance_sha256}.json", "sha256": provenance_sha256, }, }, "config": { - "context": 3, - "decode_steps": 1, + "context": context, + "decode_steps": decode_steps, "deepseek41": { "layer_count": 40, "vocab_size": 129280, @@ -511,8 +597,8 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: "logits": "byte-identical-f32", }, "expected": { - "prompt_tokens": 2, - "decode_steps": 1, + "prompt_tokens": context - decode_steps, + "decode_steps": decode_steps, "components": { "prompt.bytes": {"layers": None, "input": "tokens"}, "prompt.tokens": {"layers": None, "input": "tokens"}, @@ -579,6 +665,47 @@ def manifest(runtime: str = "llama.cpp", prompt: bytes = b"abc") -> dict: result["config"]["prefill_chunk"] = trace.ADMITTED_UBATCH result["config"]["device_backend"] = "Metal" result["config"]["device_registry_id"] = METAL_ACCELERATOR_ATTESTATION["metal_registry_id"] + prompt_policy = fixture_prompt_builder_policy( + prompt, context=context, decode_steps=decode_steps) + _prompt_policy, prompt_policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: prompt_policy}, + ) + approvals = { + "prompt_builder": trace.approval_binding( + "prompt_builder", TEST_PROMPT_BUILDER_POLICY_ID, prompt_policy_sha256), + } + if not is_ds4: + candidate_policy = { + "runtime": "llama.cpp", + "repository": trace.REPOSITORY, + "revision": result["candidate"]["revision"], + "base_revision": result["candidate"]["base_revision"], + "diff_sha256": result["candidate"]["diff_sha256"], + "install_root": "/home/repo/build", + "executable_path": result["candidate"]["executable_path"], + "executable_sha256": result["candidate"]["executable_sha256"], + "runtime_profile": copy.deepcopy(result["build"]["runtime_profile"]), + "runtime_receipt": runtime_receipt, + } + _candidate_policy, candidate_policy_sha256 = trace.candidate_exporter_approval( + TEST_CANDIDATE_EXPORTER_POLICY_ID, + policies={TEST_CANDIDATE_EXPORTER_POLICY_ID: candidate_policy}, + ) + result["candidate"]["exporter_approval_id"] = TEST_CANDIDATE_EXPORTER_POLICY_ID + result["candidate"]["exporter_approval_sha256"] = candidate_policy_sha256 + approvals["candidate_exporter"] = trace.approval_binding( + "candidate_exporter", TEST_CANDIDATE_EXPORTER_POLICY_ID, candidate_policy_sha256) + result["authorization"] = trace.execution_authorization( + lane=trace.ORACLE_LANE if is_ds4 else trace.CANDIDATE_LANE, + challenge=TEST_CHALLENGE, + run_id=TEST_RUN_IDS[runtime], + issued_unix=TEST_AUTH_ISSUED, + expires_unix=TEST_AUTH_EXPIRES, + approval_policy_sha256="e" * 64, + verifier_revision="a" * 40, + approvals=approvals, + ) return result @@ -595,7 +722,11 @@ def add_required_events(writer: object, logits: bytes | None = None, prompt: byt (audit_root / f"{WATCHDOG_JSONL_SHA256}.jsonl").write_bytes(WATCHDOG_JSONL) provenance_root = writer.root / "provenance" provenance_root.mkdir(exist_ok=True) - data = provenance_bytes(prompt) + data = provenance_bytes( + prompt, + context=writer.manifest["config"]["context"], + decode_steps=writer.manifest["config"]["decode_steps"], + ) (provenance_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) writer.add_event( component="prompt.bytes", @@ -887,12 +1018,60 @@ def _verifier_for_runtime( cls, runtime: str, *, + manifest_record: dict[str, object] | None = None, expected_challenge: str = TEST_CHALLENGE, expected_run_id: str | None = None, verification_unix: int | None = None, seen_run_ids: set[str] | None = None) -> trace.TraceVerifier: principal = cls.signer_principals[runtime] policy = cls.test_signers[principal] + manifest_record = manifest_record or manifest(runtime) + prompt_policy = fixture_prompt_builder_policy(b"abc") + prompt_policy["prompts"][0].update({ + "corpus_name": manifest_record["prompt"]["corpus_name"], + "corpus_sha256": manifest_record["prompt"]["corpus_sha256"], + "context": manifest_record["config"]["context"], + "decode_steps": manifest_record["config"]["decode_steps"], + "target_tokens": manifest_record["prompt"]["target_tokens"], + "prompt_sha256": manifest_record["prompt"]["sha256"], + "prompt_byte_count": manifest_record["prompt"]["byte_count"], + }) + prompt_policies = {TEST_PROMPT_BUILDER_POLICY_ID: prompt_policy} + candidate_policies = {} + candidate_policy_id = None + if runtime == "llama.cpp": + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": manifest_record["candidate"]["revision"], + "profile": manifest_record["build"]["runtime_profile"]["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in manifest_record["build"]["runtime_libraries"] + ], + key=lambda item: item["component"], + ), + } + candidate_policy = { + "runtime": "llama.cpp", + "repository": manifest_record["candidate"]["repository"], + "revision": manifest_record["candidate"]["revision"], + "base_revision": manifest_record["candidate"]["base_revision"], + "diff_sha256": manifest_record["candidate"]["diff_sha256"], + "install_root": "/home/repo/build", + "executable_path": manifest_record["candidate"]["executable_path"], + "executable_sha256": manifest_record["candidate"]["executable_sha256"], + "runtime_profile": copy.deepcopy(manifest_record["build"]["runtime_profile"]), + "runtime_receipt": receipt, + } + candidate_policies[TEST_CANDIDATE_EXPORTER_POLICY_ID] = candidate_policy + candidate_policy_id = TEST_CANDIDATE_EXPORTER_POLICY_ID return trace.TraceVerifier.for_tests( principal, policy["public_key"], @@ -901,6 +1080,10 @@ def _verifier_for_runtime( runtime_profile=policy["runtime_profile"], expected_challenge=expected_challenge, expected_run_id=expected_run_id or TEST_RUN_IDS[runtime], + candidate_exporter_policies=candidate_policies, + prompt_builder_policies=prompt_policies, + expected_candidate_exporter_policy_id=candidate_policy_id, + expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, verification_unix=verification_unix or int(time.time()), ssh_keygen=cls.ssh_keygen, seen_run_ids=seen_run_ids, @@ -925,6 +1108,12 @@ def test_bundle(root: Path, verify_blobs: bool = True, **_kwargs: object) -> obj runtime = manifest_record["runtime"] principal = self.signer_principals[runtime] authorization = manifest_record["authorization"] + verifier = self._verifier_for_runtime( + runtime, + manifest_record=manifest_record, + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + ) trace.seal_bundle( Path(root), private_key=self.signing_keys[runtime], @@ -932,17 +1121,19 @@ def test_bundle(root: Path, verify_blobs: bool = True, **_kwargs: object) -> obj expected_lane=authorization["lane"], expected_challenge=authorization["challenge"], expected_run_id=authorization["run_id"], + candidate_exporter_policies=verifier.candidate_exporter_policies, + prompt_builder_policies=verifier.prompt_builder_policies, + expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, + expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, + expected_verifier_revision=verifier.expected_verifier_revision, trusted_signers=self.test_signers, ssh_keygen=self.ssh_keygen, ) return self._trace_bundle_class( Path(root), verify_blobs, - verifier=self._verifier_for_runtime( - runtime, - expected_challenge=authorization["challenge"], - expected_run_id=authorization["run_id"], - ), + verifier=verifier, ) trace.TraceBundle = test_bundle @@ -960,6 +1151,12 @@ def _seal_test_bundle(self, root: Path) -> str: runtime = manifest_record["runtime"] principal = self.signer_principals[runtime] authorization = manifest_record["authorization"] + verifier = self._verifier_for_runtime( + runtime, + manifest_record=manifest_record, + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + ) return trace.seal_bundle( root, private_key=self.signing_keys[runtime], @@ -967,6 +1164,12 @@ def _seal_test_bundle(self, root: Path) -> str: expected_lane=authorization["lane"], expected_challenge=authorization["challenge"], expected_run_id=authorization["run_id"], + candidate_exporter_policies=verifier.candidate_exporter_policies, + prompt_builder_policies=verifier.prompt_builder_policies, + expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, + expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, + expected_verifier_revision=verifier.expected_verifier_revision, trusted_signers=self.test_signers, ssh_keygen=self.ssh_keygen, ) @@ -1027,7 +1230,9 @@ def test_seal_requires_external_trust_and_fixed_verifier(self) -> None: add_required_events(writer) self._seal_test_bundle(root) self._read_sealed_bundle(root) - with self.assertRaisesRegex(trace.TraceError, "external signer, lane, challenge, and run ID"): + with self.assertRaisesRegex( + trace.TraceError, + "external signer, lane, challenge, run ID, and prompt builder approval"): self._trace_bundle_class(root) with self.assertRaisesRegex(trace.TraceError, "not approved"): self._trace_bundle_class( @@ -1036,6 +1241,8 @@ def test_seal_requires_external_trust_and_fixed_verifier(self) -> None: expected_lane=trace.CANDIDATE_LANE, expected_challenge=TEST_CHALLENGE, expected_run_id=TEST_RUN_IDS["llama.cpp"], + expected_candidate_exporter_policy_id=TEST_CANDIDATE_EXPORTER_POLICY_ID, + expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, ) candidate_policy = self.test_signers[self.signer_principal] unknown = trace.TraceVerifier.for_tests( @@ -1106,6 +1313,321 @@ def test_seal_requires_external_trust_and_fixed_verifier(self) -> None: ssh_keygen=self.ssh_keygen, ) + def test_production_executable_approval_maps_fail_closed(self) -> None: + self.assertEqual(trace.APPROVED_CANDIDATE_EXPORTERS, {}) + self.assertEqual(trace.APPROVED_PROMPT_BUILDERS, {}) + self.assertEqual(trace.APPROVED_EXECUTABLE_APPROVERS, {}) + with self.assertRaisesRegex(trace.TraceError, "candidate exporter approval is not trusted"): + trace.candidate_exporter_approval(TEST_CANDIDATE_EXPORTER_POLICY_ID) + with self.assertRaisesRegex(trace.TraceError, "prompt builder approval is not trusted"): + trace.prompt_builder_approval(TEST_PROMPT_BUILDER_POLICY_ID) + + def test_external_executable_approval_signature_and_tamper(self) -> None: + verifier = self._verifier_for_runtime("llama.cpp") + principal = "dsv41-test-executable-approver" + public_key = self.test_signers[self.signer_principal]["public_key"] + policy = { + "format": trace.EXECUTABLE_APPROVAL_FORMAT, + "version": trace.EXECUTABLE_APPROVAL_VERSION, + "principal": principal, + "verifier_repository": trace.REPOSITORY, + "verifier_revision": "a" * 40, + "candidate_exporters": verifier.candidate_exporter_policies, + "prompt_builders": verifier.prompt_builder_policies, + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy_path = root / "approval.json" + policy_path.write_text(trace.canonical_json(policy) + "\n", encoding="ascii") + subprocess.run( + [ + str(self.ssh_keygen), + "-Y", "sign", + "-f", str(self.signing_key), + "-n", trace.EXECUTABLE_APPROVAL_NAMESPACE, + str(policy_path), + ], + check=True, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + signature_path = policy_path.with_suffix(".json.sig") + loaded = trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers={principal: public_key}, + ssh_keygen=self.ssh_keygen, + ) + self.assertEqual(loaded.verifier_revision, "a" * 40) + self.assertEqual(loaded.sha256, trace.sha256_file(policy_path)) + self.assertEqual( + loaded.candidate_exporters, + verifier.candidate_exporter_policies, + ) + with self.assertRaisesRegex(trace.TraceError, "outside protected output roots"): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers={principal: public_key}, + ssh_keygen=self.ssh_keygen, + forbidden_roots=(root,), + ) + tampered = copy.deepcopy(policy) + tampered["verifier_revision"] = "b" * 40 + policy_path.write_text(trace.canonical_json(tampered) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "signature verification failed"): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers={principal: public_key}, + ssh_keygen=self.ssh_keygen, + ) + with self.assertRaisesRegex(trace.TraceError, "principal is not trusted"): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers={}, + ssh_keygen=self.ssh_keygen, + ) + + def test_candidate_runner_rejects_unapproved_exporter_before_execution(self) -> None: + argv = [ + "run_llama.py", + "--exporter", "/usr/bin/true", + "--repo", "/tmp/repo", + "--candidate-revision", "a" * 40, + "--base-revision", "b" * 40, + "--candidate-diff-sha256", "c" * 64, + "--candidate-exporter-policy-id", "unapproved-exporter", + "--prompt-builder-policy-id", "unapproved-builder", + "--approval-policy", "/tmp/approval.json", + "--approval-signature", "/tmp/approval.sig", + "--approval-principal", "unapproved", + "--corpus-name", "correctness-prose.txt", + "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], + "--prompt-provenance", "/tmp/prompt.json", + "--model", "/tmp/model.gguf", + "--prompt", "/tmp/prompt.txt", + "--output", "/tmp/output", + "--signer-principal", "candidate", + "--signing-key", "/tmp/key", + "--execution-challenge", TEST_CHALLENGE, + "--run-id", TEST_RUN_IDS["llama.cpp"], + "--authorization-issued-unix", str(TEST_AUTH_ISSUED), + "--authorization-expires-unix", str(TEST_AUTH_EXPIRES), + "--preflight-only", + ] + with mock.patch.dict(os.environ, {}, clear=True), mock.patch.object( + sys, "argv", argv), mock.patch.object( + sys, "stderr", io.StringIO()), mock.patch.object( + run_llama, "query_runtime_build_attestation") as build_query, mock.patch.object( + run_llama, "query_accelerator_attestation") as accelerator_query: + self.assertEqual(run_llama.main(), 1) + build_query.assert_not_called() + accelerator_query.assert_not_called() + + def test_prompt_builder_rejects_unapproved_identity_before_execution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "actual" / "bin" / "llama-deepseek-v41-prompt-builder" + approved_builder = root / "approved" / "bin" / "llama-deepseek-v41-prompt-builder" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + policy = fixture_prompt_builder_policy( + b"prompt", + builder_path=str(approved_builder), + builder_sha256=trace.sha256_file(builder), + source_root=str(Path(__file__).parents[1].resolve()), + ) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + with mock.patch.object(run_matrix.subprocess, "run") as execute, self.assertRaisesRegex( + run_matrix.TraceError, "path differs from external approval"): + run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=policy, + builder_policy_sha256=policy_sha256, + model=root / "model.gguf", + corpus=root / "corpus.txt", + source_corpus=Path(__file__).parents[1] / "tests" / "corpus" / "correctness-prose.txt", + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=root / "prompt.txt", + context=3, + decode_steps=1, + ) + execute.assert_not_called() + + def test_approved_executable_uses_linux_descriptor_path(self) -> None: + with tempfile.TemporaryDirectory() as temp: + executable = Path(temp).resolve() / "approved" + executable.write_bytes(b"approved") + executable.chmod(0o755) + completed = subprocess.CompletedProcess([str(executable)], 0, "", "") + with mock.patch.object(trace.sys, "platform", "linux"), mock.patch.object( + trace.subprocess, "run", return_value=completed) as execute: + result, identity = trace.run_approved_executable( + [str(executable), "--version"], + path=executable, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + check=False, + capture_output=True, + text=True, + ) + self.assertIs(result, completed) + self.assertEqual(identity.path, str(executable)) + kwargs = execute.call_args.kwargs + self.assertRegex(kwargs["executable"], r"^/proc/self/fd/[0-9]+$") + self.assertEqual(len(kwargs["pass_fds"]), 1) + + def test_prompt_builder_rejects_runtime_receipt_before_execution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + policy = fixture_prompt_builder_policy( + b"prompt", + builder_path=str(builder), + builder_sha256=trace.sha256_file(builder), + source_root=str(Path(__file__).parents[1].resolve()), + ) + materialize_policy_runtime(policy) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + runtime_component = policy["runtime_receipt"]["components"][0] + (Path(policy["install_root"]) / "lib" / runtime_component["filename"]).write_bytes( + b"changed") + with mock.patch.object(run_matrix, "run_approved_executable") as execute, self.assertRaisesRegex( + run_matrix.TraceError, "runtime component .* SHA-256 differs from external approval"): + run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=policy, + builder_policy_sha256=policy_sha256, + model=root / "model.gguf", + corpus=root / "corpus.txt", + source_corpus=Path(__file__).parents[1] / "tests" / "corpus" / "correctness-prose.txt", + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=root / "prompt.txt", + context=3, + decode_steps=1, + ) + execute.assert_not_called() + + def test_prompt_builder_rejects_output_binary_and_corpus_mutation(self) -> None: + for mutation, message in ( + ("output", "output differs from external approval"), + ("builder", "SHA-256 differs from external approval"), + ("corpus", "corpus changed during execution"), + ("runtime", "runtime component SHA-256 differs from external approval"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" + model = root / "model.gguf" + corpus = root / "corpus.txt" + source_root = Path(__file__).parents[1].resolve() + source_corpus = source_root / "tests" / "corpus" / "correctness-prose.txt" + output = root / "prompt.txt" + tmpdir = root / "tmp" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + model.write_bytes(b"model") + shutil.copyfile(source_corpus, corpus) + tmpdir.mkdir() + approved_output = b"approved" if mutation == "output" else b"prompt" + policy = fixture_prompt_builder_policy( + approved_output, + builder_path=str(builder), + builder_sha256=trace.sha256_file(builder), + source_root=str(source_root), + ) + materialize_policy_runtime(policy) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + initial_identity = run_matrix.approved_executable_identity( + builder, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="prompt builder", + ) + + def run_builder(command, **_kwargs): + output.write_bytes(b"prompt") + if mutation == "builder": + builder.write_bytes(b"changed") + if mutation == "corpus": + corpus.write_bytes(b"changed") + if mutation == "runtime": + runtime_component = policy["runtime_receipt"]["components"][0] + (Path(policy["install_root"]) / "lib" / runtime_component["filename"]).write_bytes( + b"changed") + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps({ + "target_tokens": 2, + "actual_tokens": 2, + "byte_count": 6, + "add_bos": True, + "temporary_directory": str(tmpdir), + }), + "", + ), + initial_identity, + ) + + with mock.patch.dict(os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( + run_matrix, "run_approved_executable", side_effect=run_builder), self.assertRaisesRegex( + (RuntimeError, run_matrix.TraceError), message): + run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=policy, + builder_policy_sha256=policy_sha256, + model=model, + corpus=corpus, + source_corpus=source_corpus, + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=output, + context=3, + decode_steps=1, + ) + + def test_signed_approval_binding_tamper_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + trace_manifest = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + trace_manifest["authorization"]["approvals"]["candidate_exporter"]["sha256"] = "f" * 64 + (root / trace.MANIFEST_NAME).write_text( + trace.canonical_json(trace_manifest) + "\n", encoding="ascii") + with self.assertRaisesRegex( + trace.TraceError, "candidate exporter approval differs from external policy"): + self._seal_test_bundle(root) + def test_seal_rejects_protected_bundle_mutations(self) -> None: with tempfile.TemporaryDirectory() as temp: baseline = Path(temp) / "baseline" @@ -2544,20 +3066,20 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: short_revision_manifest = manifest() short_revision_manifest["revision"] = "a" * 9 short_revision_manifest["candidate"]["revision"] = "a" * 9 - cases.append((short_revision_manifest, "exact full Git revision")) + cases.append((short_revision_manifest, "candidate exporter approval revision")) revision_manifest = manifest() revision_manifest["candidate"]["revision"] = "d" * 40 - cases.append((revision_manifest, "candidate revision")) + cases.append((revision_manifest, "candidate exporter approval")) executable_manifest = manifest() executable_manifest["candidate"]["executable_path"] = "/home/repo/build/bin/other-exporter" - cases.append((executable_manifest, "candidate executable path")) + cases.append((executable_manifest, "candidate exporter approval executable path")) library_manifest = manifest() library_manifest["build"]["runtime_libraries"][0]["sha256"] = "e" * 64 library_manifest["build"]["runtime_libraries_post"][0]["sha256"] = "e" * 64 - cases.append((library_manifest, "runtime receipt SHA-256")) + cases.append((library_manifest, "candidate exporter approval")) added_module_manifest = manifest() added_module_manifest["build"]["runtime_module_monitor"]["project_additions"] = [ @@ -2591,7 +3113,7 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: ] omitted_library_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( omitted_library_manifest["build"]["runtime_libraries"]) - cases.append((omitted_library_manifest, "set differs from the runtime profile")) + cases.append((omitted_library_manifest, "candidate exporter approval receipt")) duplicate_path_manifest = manifest() duplicate_path_manifest["build"]["runtime_libraries"][1]["path"] = ( @@ -2610,7 +3132,7 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: unknown_component_manifest = manifest() unknown_component_manifest["build"]["runtime_libraries"][0]["component"] = "ggml-injected" unknown_component_manifest["build"]["runtime_libraries_post"][0]["component"] = "ggml-injected" - cases.append((unknown_component_manifest, "runtime library component is invalid")) + cases.append((unknown_component_manifest, "candidate exporter approval runtime receipt component")) revision_library_manifest = manifest() revision_library = next( @@ -2622,7 +3144,7 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: library for library in revision_library_manifest["build"]["runtime_libraries_post"] if library["role"] == "build-info")["revision"] = "b" * 40 - cases.append((revision_library_manifest, "runtime library revision is invalid")) + cases.append((revision_library_manifest, "candidate exporter approval runtime receipt revision")) unexpected_revision_manifest = manifest() unexpected_revision = next( @@ -2634,7 +3156,7 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: library for library in unexpected_revision_manifest["build"]["runtime_libraries_post"] if library["role"].startswith("runtime:"))["revision"] = "a" * 40 - cases.append((unexpected_revision_manifest, "runtime library revision is unexpected")) + cases.append((unexpected_revision_manifest, "candidate exporter approval runtime receipt revision")) for trace_manifest, message in cases: with self.subTest(message=message), tempfile.TemporaryDirectory() as temp: @@ -2957,6 +3479,10 @@ def test_native_complete_manifest_writer_validates(self) -> None: native["accelerator"], manifest_binary, trace.sha256_file(manifest_binary), + { + "runtime_profile": native["build"]["runtime_profile"], + "runtime_receipt": receipt, + }, ) with self.assertRaisesRegex(trace.TraceError, "execution authorization is missing"): trace.seal_bundle( @@ -3070,11 +3596,16 @@ def test_runtime_build_validator_rejects_closure_substitution(self) -> None: }, }, } + approval = { + "runtime_profile": copy.deepcopy(build_manifest["build"]["runtime_profile"]), + "runtime_receipt": copy.deepcopy(receipt), + } libraries_digest, receipt_digest = run_llama.validate_runtime_build( build_manifest, exporter=exporter, exporter_sha256=trace.sha256_file(exporter), candidate_revision="a" * 40, + approval=approval, ) self.assertEqual( libraries_digest, @@ -3198,6 +3729,7 @@ def test_runtime_build_validator_rejects_closure_substitution(self) -> None: exporter=exporter, exporter_sha256=trace.sha256_file(exporter), candidate_revision="a" * 40, + approval=approval, ) @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "loader injection test") @@ -3245,50 +3777,87 @@ def test_python_runner_rejects_loader_overrides(self) -> None: def test_prompt_builder_result_becomes_strict_provenance(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) - builder = root / "prompt-builder" + builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" model = root / "model.gguf" corpus = root / "corpus.txt" + source_root = Path(__file__).parents[1] + source_corpus = source_root / "tests" / "corpus" / "correctness-prose.txt" output = root / "prompt.txt" tmpdir = root / "tmp" + builder.parent.mkdir(parents=True) builder.write_bytes(b"builder") + builder.chmod(0o755) model.write_bytes(b"model") - corpus.write_bytes(b"corpus") + shutil.copyfile(source_corpus, corpus) tmpdir.mkdir() + builder = builder.resolve() + model = model.resolve() + corpus = corpus.resolve() + output = output.resolve() + tmpdir = tmpdir.resolve() + builder_policy = fixture_prompt_builder_policy( + b"prompt", + builder_path=str(builder.resolve()), + builder_sha256=trace.sha256_file(builder), + source_root=str(source_root.resolve()), + ) + materialize_policy_runtime(builder_policy) + _validated, builder_policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: builder_policy}, + ) + builder_identity = run_matrix.approved_executable_identity( + builder, + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + ) def run_builder(command, **_kwargs): output.write_bytes(b"prompt") - return subprocess.CompletedProcess( - command, - 0, - json.dumps({ - "target_tokens": 2, - "actual_tokens": 2, - "byte_count": 6, - "add_bos": True, - "temporary_directory": str(tmpdir.resolve()), - }), - "", + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps({ + "target_tokens": 2, + "actual_tokens": 2, + "byte_count": 6, + "add_bos": True, + "temporary_directory": str(tmpdir.resolve()), + }), + "", + ), + builder_identity, ) with mock.patch.dict(os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( - run_matrix.subprocess, "run", side_effect=run_builder), mock.patch.object( + run_matrix, "run_approved_executable", side_effect=run_builder), mock.patch.object( sys, "stderr", io.StringIO()): result = run_matrix.prepare_prompt( builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=builder_policy, + builder_policy_sha256=builder_policy_sha256, model=model, corpus=corpus, + source_corpus=source_corpus, corpus_name="correctness-prose.txt", corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], output=output, - target_tokens=2, + context=3, + decode_steps=1, ) provenance_path = Path(result["provenance_path"]) provenance = trace.strict_json_loads(provenance_path.read_text(encoding="ascii")) self.assertEqual( set(provenance), { - "format", "version", "corpus_name", "corpus_sha256", "model_sha256", - "prompt_sha256", "prompt_byte_count", "builder_sha256", "target_tokens", "actual_tokens", + "format", "version", "corpus_name", "corpus_sha256", "corpus_path", + "model_sha256", "prompt_sha256", "prompt_byte_count", "context", + "decode_steps", "builder_approval_id", "builder_approval_sha256", + "builder_path", "builder_sha256", "builder_revision", + "builder_runtime_profile", "target_tokens", "actual_tokens", }, ) preflight.validate_prompt_provenance( @@ -3297,7 +3866,12 @@ def run_builder(command, **_kwargs): corpus_name="correctness-prose.txt", corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], model_sha256=trace.MODEL_SHA256, + context=3, + decode_steps=1, target_tokens=2, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=builder_policy, + builder_policy_sha256=builder_policy_sha256, path_resolver=lambda path, _label: path.resolve(), ) @@ -3545,6 +4119,10 @@ def test_unapproved_ds4_exporter_is_not_executed(self) -> None: "--corpus-name", "correctness-prose.txt", "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], "--prompt-provenance", str(root / "prompt.json"), + "--prompt-builder-policy-id", TEST_PROMPT_BUILDER_POLICY_ID, + "--approval-policy", str(root / "approval.json"), + "--approval-signature", str(root / "approval.sig"), + "--approval-principal", "unapproved", "--signer-principal", self.signer_principals["ds4"], "--signing-key", str(self.signing_keys["ds4"]), "--execution-challenge", TEST_CHALLENGE, @@ -3949,6 +4527,20 @@ def test_local_base_regression_requires_attested_oracle_revision(self) -> None: "post": base_manifest["build"]["runtime_libraries_post"], }).encode("ascii")) base_manifest["candidate"]["runtime_receipt_sha256"] = receipt_sha256 + base_verifier = self._verifier_for_runtime( + "llama.cpp", manifest_record=base_manifest) + _candidate_policy, candidate_policy_sha256 = trace.candidate_exporter_approval( + TEST_CANDIDATE_EXPORTER_POLICY_ID, + policies=base_verifier.candidate_exporter_policies, + ) + base_manifest["candidate"]["exporter_approval_sha256"] = candidate_policy_sha256 + base_manifest["authorization"]["approvals"]["candidate_exporter"] = ( + trace.approval_binding( + "candidate_exporter", + TEST_CANDIDATE_EXPORTER_POLICY_ID, + candidate_policy_sha256, + ) + ) with trace.TraceBundleWriter(base, base_manifest) as writer: add_required_events(writer) with trace.TraceBundleWriter(integrated, manifest("llama.cpp")) as writer: @@ -3978,10 +4570,7 @@ def test_manifest_mismatch_is_classified(self) -> None: def test_identically_incomplete_decode_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "trace" - incomplete = manifest() - incomplete["config"]["context"] = 4 - incomplete["config"]["decode_steps"] = 2 - incomplete["expected"]["decode_steps"] = 2 + incomplete = manifest(context=4, decode_steps=2) with trace.TraceBundleWriter(root, incomplete) as writer: add_required_events(writer) with self.assertRaisesRegex(trace.TraceError, "decode step coverage"): diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 301dbcaba10b..6e7075538b97 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -15,6 +15,10 @@ Seal v1 signs `dsv41-trace-bundle-v1\n` followed by canonical JSON records for ` Signer trust is external to the bundle. `APPROVED_TRACE_SIGNERS` maps one restricted ASCII principal to one exact OpenSSH Ed25519 public key, runtime lane, and runtime profile, and is intentionally empty until a separately authorized run. Candidate and oracle signers are not interchangeable. Every signed manifest also binds an externally supplied 256-bit challenge, lane-specific run ID, and bounded validity window. Validation requires those expected values from outside the bundle and rejects missing, mismatched, reused within a comparison, cross-lane, not-yet-valid, or expired authorization. The bundle cannot provide a public key, authoritative principal, verifier path, or allowed-signers file. Validation uses only `/usr/bin/ssh-keygen` on macOS and Linux or `C:\Windows\System32\OpenSSH\ssh-keygen.exe` on Windows, rejects symlinks and unsupported `-Y` implementations, clears SSH-agent influence, and never searches `PATH`. The private signing key must be an owned restrictive regular file outside the bundle and is never copied or logged. Tests use explicit test-only verifiers with ephemeral lane-specific keys; production validation does not trust those keys. +Executable trust is also external to the bundle and to the reviewed source revision. `APPROVED_EXECUTABLE_APPROVERS`, `APPROVED_CANDIDATE_EXPORTERS`, and `APPROVED_PROMPT_BUILDERS` are intentionally empty in production source. An authorized run must receive a canonical detached approval policy plus its OpenSSH signature and an externally expected approver principal. Both files must be absolute canonical non-symlinked regular files outside every protected output root. The approver public key comes only from `APPROVED_EXECUTABLE_APPROVERS`; the fixed `ssh-keygen` verifies namespace `dsv41-executable-approval-v1` before either executable can run. This avoids an impossible same-revision self-reference: the policy records the artifact producer revision and exact executable hashes, while a separate `verifier_revision` records the harness revision that consumes the policy. + +The candidate exporter approval binds the exact producer revision, base revision, binary diff SHA-256, canonical install root and exporter path, exporter SHA-256, runtime profile, and complete embedded runtime receipt. The prompt-builder approval binds its producer revision, canonical install/source roots and executable path, executable SHA-256, exact runtime profile and receipt, model and corpus identities, and exact prompt hash, byte count, BOS behavior, context, and decode configuration for every authorized case. Every receipt library is hashed before the corresponding executable can run and rechecked afterward. On the Linux candidate host, both the exporter and prompt builder are opened without following the final symlink and executed through the retained `/proc/self/fd` descriptor, so pathname replacement cannot select a different executable object. The signed execution authorization records the complete approval-policy SHA-256, verifier revision, and both selected approval record IDs and hashes. Candidate-derived attestations and prompt provenance are evidence only and must exactly equal those external records. + Every manifest and memory audit declares that the expert cache and KV cache are memory-resident and that there are no external cache or state paths. Missing, substituted, or additional file-backed cache/state declarations fail closed. The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. @@ -28,11 +32,21 @@ Validate or compare bundles: ```sh python3 tools/deepseek-v41-trace/trace_format.py validate TRACE \ --signer-principal PRINCIPAL --lane strix-llama-candidate-v1 \ - --execution-challenge "$CHALLENGE" --run-id "$RUN_ID" + --execution-challenge "$CHALLENGE" --run-id "$RUN_ID" \ + --candidate-exporter-policy-id "$CANDIDATE_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_SIGNATURE" \ + --approval-principal "$APPROVAL_PRINCIPAL" python3 tools/deepseek-v41-trace/trace_format.py compare DS4_TRACE LLAMA_TRACE \ --left-signer-principal DS4_PRINCIPAL --right-signer-principal LLAMA_PRINCIPAL \ --execution-challenge "$CHALLENGE" \ --left-run-id "$DS4_RUN_ID" --right-run-id "$LLAMA_RUN_ID" \ + --right-candidate-exporter-policy-id "$CANDIDATE_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_SIGNATURE" \ + --approval-principal "$APPROVAL_PRINCIPAL" \ --report report.json ``` @@ -142,13 +156,13 @@ The exporter is intentionally external to the canonical ds4 checkout. It must be The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. The content-addressed memory audit binds the preflight accelerator and storage attestations; the manifest binds the independently repeated native accelerator attestation. -Use `run_llama.py` on the validation host instead of calling the exporter directly. It applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact final integration revision, immutable oracle revision, expected oracle-to-candidate binary diff SHA-256, repository path, externally approved signer principal, and matching private signing key. The private key is never copied or logged. Its public half is derived with the fixed trusted `ssh-keygen` executable and must exactly match the source-controlled signer map before model execution. +Use `run_llama.py` on the validation host instead of calling the exporter directly. It verifies the detached executable approval policy before invoking the exporter, then applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact producer revision, immutable base revision, expected base-to-producer binary diff SHA-256, verifier repository path and revision, external executable approval, externally approved signer principal, and matching private signing key. The private key is never copied or logged. Its public half is derived with the fixed trusted `ssh-keygen` executable and must exactly match the source-controlled signer map before model execution. -The native exporter embeds the full 40-character candidate revision independently of dynamically loaded build-info, resolves its actual executable path, and validates every loaded `llama`, `ggml`, and enabled backend project library against the build-generated component receipt and selected runtime profile. Component name, filename, canonical path, SHA-256, role, and exact revision-bearing identity must match, and the measured loaded set must equal the predeclared profile set both immediately before protected trace generation and after it completes. Both snapshots and the completed loader-monitor receipt are signed. macOS also monitors loader additions during the protected interval. Missing, duplicated, unclassified, outside-root, renamed inside-root injected, catalogued-but-not-profile, changed, or late-loaded project libraries fail closed. Linux enumerates loaded ELF objects rather than arbitrary memory mappings and accepts non-project ROCm dependencies only from a root-owned, non-writable `/opt/rocm` installation. Production launchers and the native exporter reject dynamic-loader and `GGML_BACKEND_PATH` overrides. The executable hash is bound separately by the signed manifest to avoid link-time hash circularity. +The launcher checks the approved candidate exporter path, hash, device/inode, size, modification time, and change time before any exporter invocation and after each protected operation. The read-only `--dsv41-attest-build ROCm0` command loads the selected backend and emits the embedded runtime profile and receipt before model execution; the launcher requires exact equality with the external approval and repeats the attestation after trace completion. The native exporter embeds the full 40-character producer revision independently of dynamically loaded build-info, resolves its actual executable path, and validates every loaded `llama`, `ggml`, and enabled backend project library against the build-generated component receipt and selected runtime profile. Component name, filename, canonical path, SHA-256, role, and exact revision-bearing identity must match, and the measured loaded set must equal the predeclared profile set both immediately before protected trace generation and after it completes. Both snapshots and the completed loader-monitor receipt are signed. macOS also monitors loader additions during the protected interval. Missing, duplicated, unclassified, outside-root, renamed inside-root injected, catalogued-but-not-profile, changed, or late-loaded project libraries fail closed. Linux enumerates loaded ELF objects rather than arbitrary memory mappings and accepts non-project ROCm dependencies only from a root-owned, non-writable `/opt/rocm` installation. Production launchers and the native exporter reject dynamic-loader and `GGML_BACKEND_PATH` overrides. Production installs no manifest-writing or runtime-path probe option. With `LLAMA_BUILD_TESTS`, CMake builds the non-installed `test-deepseek41-trace-manifest` harness from the same native writer implementation. Its input accepts only model, prompt, audit, expected-coverage, and event-count data. Runtime, accelerator, path, configuration, build, and environment evidence comes from measured local state, and CPU or Darwin output is explicitly test-only and cannot satisfy production candidate or signer requirements. The trace install component places the exact receipt libraries beside the tools and sets `@loader_path/../lib` on macOS or `$ORIGIN/../lib` on ELF so installed `--version` needs no loader override. The install test also hashes every installed component and requires exact equality with the receipt embedded after the original library link; install-time rewriting or re-signing fails. -`run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, builds exact-length prompt artifacts and content-addressed provenance, and captures the llama.cpp side. Pass both `--llama-exporter` and `--llama-prompt-builder` from the same build, plus the final integration revision, immutable oracle revision, and expected binary diff SHA-256. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. +`run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, and verifies the approved builder path/hash and both original and copied corpus identities before prompt construction. It rechecks the builder and corpus identities after execution and requires the generated prompt hash, byte count, BOS behavior, context, and decode configuration to equal the signed external approval before writing provenance. Pass the detached approval policy and signature, selected candidate and prompt approval IDs, both executables, the producer revision/base/diff identity, and the verifier checkout. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. ## Apple Metal oracle execution gate @@ -179,6 +193,11 @@ DIFF_SHA256="$(git -C "$REPO" diff --binary --no-ext-diff "$BASE_REV" "$CANDIDAT CHALLENGE=<64-lowercase-hex-execution-challenge> AUTH_ISSUED= AUTH_EXPIRES= +APPROVAL_POLICY= +APPROVAL_SIGNATURE= +APPROVAL_PRINCIPAL= +CANDIDATE_EXPORTER_POLICY_ID= +PROMPT_BUILDER_POLICY_ID= mkdir -p "$CASE_ROOT/watchdog" cd "$REPO" @@ -203,6 +222,11 @@ HIP_LAUNCH_BLOCKING=1 python3 scripts/strix_memory_watchdog.py \ --candidate-revision "$CANDIDATE_REV" \ --base-revision "$BASE_REV" \ --candidate-diff-sha256 "$DIFF_SHA256" \ + --candidate-exporter-policy-id "$CANDIDATE_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_SIGNATURE" \ + --approval-principal "$APPROVAL_PRINCIPAL" \ --signer-principal "$LLAMA_SIGNER_PRINCIPAL" \ --signing-key "$LLAMA_SIGNING_KEY" \ --execution-challenge "$CHALLENGE" \ @@ -239,6 +263,10 @@ python3 tools/deepseek-v41-trace/run_ds4.py \ --prompt /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt \ --prompt-provenance /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt.provenance.json \ --output /Users/oracle/dsv41/traces/correctness-prose-c32768-ub32 \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-policy-signature "$APPROVAL_POLICY_SIGNATURE" \ + --approval-approver-principal "$APPROVAL_APPROVER_PRINCIPAL" \ + --prompt-builder-approval-id "$PROMPT_BUILDER_APPROVAL_ID" \ --corpus-name correctness-prose.txt \ --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ --context 32768 \ diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index a01ac7ccedcf..c16f2725199c 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -1447,6 +1447,20 @@ int main(int argc, char ** argv) { << " for " << build["target"].get() << '\n'; return 0; } + if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-build") { + if (std::string(argv[2]) != "ROCm0") { + throw std::runtime_error("build attestation requires selected execution device ROCm0"); + } + common_init(); + const fs::path executable = current_executable_path(); + load_runtime_backends(executable); + ggml_backend_dev_t device = ggml_backend_dev_by_name(argv[2]); + if (device == nullptr) { + throw std::runtime_error("cannot find selected execution device ROCm0"); + } + std::cout << runtime_build_json(executable, device, "ggml-hip", argv).dump() << '\n'; + return 0; + } if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { common_init(); load_runtime_backends(current_executable_path()); diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index c3ed40a6120f..3511063aa9ff 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -1339,6 +1339,11 @@ def validate_prompt_provenance( corpus_sha256: str, model_sha256: str, target_tokens: int, + context: int, + decode_steps: int, + builder_approval_id: str, + builder_policy: dict[str, object], + builder_policy_sha256: str, path_resolver: Callable[[Path, str], Path] | None = None, ) -> dict[str, object]: path = (require_nvme_path if path_resolver is None else path_resolver)(path, "prompt provenance") @@ -1357,21 +1362,39 @@ def validate_prompt_provenance( "version": 1, "corpus_name": corpus_name, "corpus_sha256": corpus_sha256, + "corpus_path": f"{builder_policy['source_root']}/tests/corpus/{corpus_name}", "model_sha256": model_sha256, "prompt_sha256": sha256_bytes(prompt_bytes), "prompt_byte_count": prompt_size, + "context": context, + "decode_steps": decode_steps, "target_tokens": target_tokens, "actual_tokens": target_tokens, + "builder_approval_id": builder_approval_id, + "builder_approval_sha256": builder_policy_sha256, + "builder_path": builder_policy["executable_path"], + "builder_sha256": builder_policy["executable_sha256"], + "builder_revision": builder_policy["revision"], + "builder_runtime_profile": builder_policy["runtime_profile"], } for key, value in expected.items(): if record.get(key) != value: raise PreflightError(f"prompt provenance {key} mismatch") - builder_sha256 = record.get("builder_sha256", "") - if not isinstance(builder_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", builder_sha256) is None: - raise PreflightError("prompt provenance builder SHA-256 is invalid") - required_keys = set(expected) | {"builder_sha256"} - if set(record) != required_keys: + if set(record) != set(expected): raise PreflightError("prompt provenance fields are invalid") + matches = [ + prompt_record for prompt_record in builder_policy["prompts"] + if prompt_record["corpus_name"] == corpus_name and + prompt_record["context"] == context and + prompt_record["decode_steps"] == decode_steps + ] + if len(matches) != 1: + raise PreflightError("prompt provenance configuration is not externally approved") + for key in ( + "corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", + "prompt_sha256", "prompt_byte_count"): + if record[key] != matches[0][key]: + raise PreflightError(f"prompt provenance {key} differs from external approval") return {"path": str(path), "bytes": data, "record": record} diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index e89860b23444..05670e03929b 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -23,6 +23,7 @@ from trace_format import ( ADMITTED_UBATCH, APPROVED_EXPORTERS, + APPROVED_PROMPT_BUILDERS, APPROVED_TRACE_SIGNERS, CORPUS_SHA256, DS4_REVISION, @@ -32,9 +33,12 @@ TraceBundle, TraceError, TraceVerifier, + approval_binding, bind_execution_authorization, canonical_json, execution_authorization, + load_executable_approval_policy, + prompt_builder_approval, reject_loader_overrides, seal_bundle, sha256_bytes, @@ -285,6 +289,10 @@ def main() -> int: parser.add_argument("--run-id", required=True) parser.add_argument("--authorization-issued-unix", type=int, required=True) parser.add_argument("--authorization-expires-unix", type=int, required=True) + parser.add_argument("--prompt-builder-policy-id", required=True) + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() @@ -294,14 +302,40 @@ def main() -> int: f"DeepSeek V4.1 correctness runs require admitted prefill chunk {ADMITTED_UBATCH}, " f"found {args.prefill_chunk}") reject_loader_overrides() + output = resolved(args.output) + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(output,), + ) + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + args.prompt_builder_policy_id, + policies=approval_policy.prompt_builders, + ) authorization = execution_authorization( lane=ORACLE_LANE, challenge=args.execution_challenge, run_id=args.run_id, issued_unix=args.authorization_issued_unix, expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + approvals={ + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + ), + }, ) - output = resolved(args.output) + harness_repo = resolved(args.repo) + harness_revision = subprocess.check_output( + ["git", "-C", str(harness_repo), "rev-parse", "HEAD"], + stderr=subprocess.STDOUT, + ).decode("ascii").strip() + if harness_revision != approval_policy.verifier_revision: + raise PreflightError("ds4 verifier checkout differs from the external approval policy") if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") exporter = resolved(args.exporter) @@ -350,6 +384,11 @@ def main() -> int: corpus_sha256=args.corpus_sha256, model_sha256=model_sha256, target_tokens=args.context - args.decode_steps, + context=args.context, + decode_steps=args.decode_steps, + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, path_resolver=lambda path, label: Path( str(darwin_storage_attestation(path, label)["resolved_path"])), ) @@ -382,6 +421,12 @@ def main() -> int: expected_lane=ORACLE_LANE, expected_challenge=args.execution_challenge, expected_run_id=args.run_id, + candidate_exporter_policies={}, + prompt_builder_policies=approval_policy.prompt_builders, + expected_candidate_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + expected_approval_policy_sha256=approval_policy.sha256, + expected_verifier_revision=approval_policy.verifier_revision, trusted_signers=APPROVED_TRACE_SIGNERS, ) bundle = TraceBundle( @@ -391,6 +436,10 @@ def main() -> int: expected_lane=ORACLE_LANE, expected_challenge=args.execution_challenge, expected_run_id=args.run_id, + expected_candidate_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, ), ) if bundle.manifest.get("runtime") != "ds4": diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index f03bf0431850..b4f4eab2cff7 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import copy import hashlib import json import os @@ -25,6 +26,8 @@ from trace_format import ( ADMITTED_BATCH, ADMITTED_UBATCH, + APPROVED_CANDIDATE_EXPORTERS, + APPROVED_PROMPT_BUILDERS, APPROVED_TRACE_SIGNERS, CANDIDATE_LANE, CORPUS_SHA256, @@ -36,15 +39,24 @@ TraceBundle, TraceError, TraceVerifier, + approval_binding, + approved_executable_identity, + approved_runtime_file_identities, bind_execution_authorization, + candidate_exporter_approval, canonical_json, execution_authorization, + load_executable_approval_policy, reject_loader_overrides, + run_approved_executable, seal_bundle, sha256_bytes, sha256_file, strict_json_loads, + prompt_builder_approval, validate_signing_identity, + verify_approved_executable_identity, + verify_approved_runtime_file_identities, ) @@ -58,11 +70,19 @@ def git_output(repo: Path, *args: str) -> bytes: def candidate_attestation( args: argparse.Namespace, exporter: Path, - exporter_sha256: str) -> dict[str, str]: + exporter_sha256: str, + approval_id: str, + approval_sha256: str, + approval: dict[str, object], + verifier_revision: str) -> dict[str, str]: repo = resolved(args.repo) exporter = resolved(exporter) - revision = git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() + observed_verifier_revision = git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() + revision = git_output(repo, "rev-parse", args.candidate_revision).decode("ascii").strip() base_revision = git_output(repo, "rev-parse", args.base_revision).decode("ascii").strip() + if observed_verifier_revision != verifier_revision: + raise PreflightError( + f"verifier revision mismatch: expected {verifier_revision}, found {observed_verifier_revision}") if revision != args.candidate_revision: raise PreflightError( f"candidate revision mismatch: expected {args.candidate_revision}, found {revision}") @@ -98,7 +118,7 @@ def candidate_attestation( if diff_sha256 != args.candidate_diff_sha256: raise PreflightError( f"candidate diff SHA-256 mismatch: expected {args.candidate_diff_sha256}, found {diff_sha256}") - return { + expected = { "repository": REPOSITORY, "revision": revision, "base_revision": base_revision, @@ -106,6 +126,14 @@ def candidate_attestation( "executable_path": str(exporter), "executable_sha256": exporter_sha256, } + for key, value in expected.items(): + if approval.get(key) != value: + raise PreflightError(f"candidate {key} differs from external exporter approval") + return { + **expected, + "exporter_approval_id": approval_id, + "exporter_approval_sha256": approval_sha256, + } def validate_runtime_build( @@ -113,7 +141,8 @@ def validate_runtime_build( *, exporter: Path, exporter_sha256: str, - candidate_revision: str) -> tuple[str, str]: + candidate_revision: str, + approval: dict[str, object]) -> tuple[str, str]: build = manifest.get("build") if not isinstance(build, dict): raise PreflightError("llama trace build identity is missing") @@ -227,16 +256,63 @@ def validate_runtime_build( receipt_sha256 = sha256_bytes(canonical_json(receipt).encode("ascii")) if build.get("runtime_receipt_sha256") != receipt_sha256: raise PreflightError("llama trace runtime receipt SHA-256 mismatch") + if approval.get("runtime_profile") != profile or approval.get("runtime_receipt") != receipt: + raise PreflightError("llama trace runtime receipt differs from external exporter approval") closure = {"pre": libraries, "post": post_libraries} return sha256_bytes(canonical_json(closure).encode("ascii")), receipt_sha256 +def query_runtime_build_attestation( + exporter: Path, + device: str, + *, + exporter_sha256: str, + candidate_revision: str, + approval: dict[str, object]) -> dict[str, object]: + result, _identity = run_approved_executable( + [str(exporter), "--dsv41-attest-build", device], + path=exporter, + expected_path=approval["executable_path"], + expected_sha256=approval["executable_sha256"], + label="candidate exporter", + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise PreflightError(f"candidate exporter build attestation failed: {result.stderr.strip()}") + try: + build = strict_json_loads(result.stdout) + except TraceError as error: + raise PreflightError(f"candidate exporter build attestation is invalid: {error}") from error + if not isinstance(build, dict): + raise PreflightError("candidate exporter build attestation is not an object") + manifest = {"revision": candidate_revision, "build": copy.deepcopy(build)} + libraries = manifest["build"].get("runtime_libraries") + if not isinstance(libraries, list): + raise PreflightError("candidate exporter build attestation has no runtime libraries") + manifest["build"]["runtime_libraries_post"] = copy.deepcopy(libraries) + monitor = manifest["build"].get("runtime_module_monitor") + if not isinstance(monitor, dict): + raise PreflightError("candidate exporter build attestation has no runtime monitor") + monitor["checked_after_trace"] = True + validate_runtime_build( + manifest, + exporter=exporter, + exporter_sha256=exporter_sha256, + candidate_revision=candidate_revision, + approval=approval, + ) + return build + + def bind_candidate_attestation( output: Path, attestation: dict[str, str], accelerator: dict[str, object], exporter: Path, - exporter_sha256: str) -> None: + exporter_sha256: str, + approval: dict[str, object]) -> None: manifest_path = safe_trace_path(output, "manifest.json") try: manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) @@ -250,6 +326,7 @@ def bind_candidate_attestation( exporter=exporter, exporter_sha256=exporter_sha256, candidate_revision=attestation["revision"], + approval=approval, ) bound_attestation["runtime_libraries_sha256"] = libraries_sha256 bound_attestation["runtime_receipt_sha256"] = receipt_sha256 @@ -309,14 +386,29 @@ def validate_accelerator_attestation( return dict(record) -def query_accelerator_attestation(exporter: Path, device: str) -> dict[str, object]: +def query_accelerator_attestation( + exporter: Path, + device: str, + approval: dict[str, object] | None = None) -> dict[str, object]: try: - result = subprocess.run( - [str(exporter), "--dsv41-attest-device", device], - check=False, - capture_output=True, - text=True, - ) + if approval is None: + result = subprocess.run( + [str(exporter), "--dsv41-attest-device", device], + check=False, + capture_output=True, + text=True, + ) + else: + result, _identity = run_approved_executable( + [str(exporter), "--dsv41-attest-device", device], + path=exporter, + expected_path=approval["executable_path"], + expected_sha256=approval["executable_sha256"], + label="candidate exporter", + check=False, + capture_output=True, + text=True, + ) except OSError as error: raise PreflightError(f"cannot query selected accelerator: {error}") from error if result.returncode != 0: @@ -378,6 +470,11 @@ def main() -> int: parser.add_argument("--candidate-revision", required=True) parser.add_argument("--base-revision", required=True) parser.add_argument("--candidate-diff-sha256", required=True) + parser.add_argument("--candidate-exporter-policy-id", required=True) + parser.add_argument("--prompt-builder-policy-id", required=True) + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) parser.add_argument("--corpus-sha256", required=True) parser.add_argument("--prompt-provenance", type=Path, required=True) @@ -405,14 +502,42 @@ def main() -> int: try: validate_runtime_config(args) reject_loader_overrides() + output = resolved(args.output) + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(output,), + ) + candidate_policy, candidate_policy_sha256 = candidate_exporter_approval( + args.candidate_exporter_policy_id, + policies=approval_policy.candidate_exporters, + ) + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + args.prompt_builder_policy_id, + policies=approval_policy.prompt_builders, + ) authorization = execution_authorization( lane=CANDIDATE_LANE, challenge=args.execution_challenge, run_id=args.run_id, issued_unix=args.authorization_issued_unix, expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + approvals={ + "candidate_exporter": approval_binding( + "candidate_exporter", + args.candidate_exporter_policy_id, + candidate_policy_sha256, + ), + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + ), + }, ) - output = resolved(args.output) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") validate_signing_identity( @@ -421,10 +546,42 @@ def main() -> int: trusted_signers=APPROVED_TRACE_SIGNERS, forbidden_root=output, ) - exporter = resolved(args.exporter) - if not exporter.is_file() or not os.access(exporter, os.X_OK): - raise PreflightError(f"trace exporter is not executable: {exporter}") - accelerator = query_accelerator_attestation(exporter, args.device) + exporter = args.exporter + exporter_identity = approved_executable_identity( + exporter, + expected_path=candidate_policy["executable_path"], + expected_sha256=candidate_policy["executable_sha256"], + label="candidate exporter", + ) + runtime_identities = approved_runtime_file_identities( + candidate_policy, label="candidate exporter") + exporter_sha256 = exporter_identity.sha256 + if args.candidate_revision != candidate_policy["revision"] or ( + args.base_revision != candidate_policy["base_revision"]) or ( + args.candidate_diff_sha256 != candidate_policy["diff_sha256"]): + raise PreflightError("candidate arguments differ from external exporter approval") + repo = resolved(args.repo) + if git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() != ( + approval_policy.verifier_revision): + raise PreflightError("candidate verifier checkout differs from the external approval policy") + if str(repo) != prompt_policy["source_root"] or candidate_policy["revision"] != prompt_policy["revision"]: + raise PreflightError("candidate repository or revision differs from prompt builder approval") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + pre_runtime_build = query_runtime_build_attestation( + exporter, + args.device, + exporter_sha256=exporter_sha256, + candidate_revision=args.candidate_revision, + approval=candidate_policy, + ) + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + accelerator = query_accelerator_attestation(exporter, args.device, candidate_policy) + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") if args.preflight_only: audit = run_strix_preflight( model=args.model, @@ -437,7 +594,6 @@ def main() -> int: print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) return 0 - exporter_sha256 = sha256_file(exporter) model_sha256 = sha256_file(resolved(args.model)) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") @@ -448,8 +604,21 @@ def main() -> int: corpus_sha256=args.corpus_sha256, model_sha256=model_sha256, target_tokens=args.context - args.decode_steps, + context=args.context, + decode_steps=args.decode_steps, + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, + ) + attestation = candidate_attestation( + args, + exporter, + exporter_sha256, + args.candidate_exporter_policy_id, + candidate_policy_sha256, + candidate_policy, + approval_policy.verifier_revision, ) - attestation = candidate_attestation(args, exporter, exporter_sha256) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") preflight_audit = run_strix_preflight( @@ -481,9 +650,37 @@ def main() -> int: environment["DSV41_TRACE_WATCHDOG_AUDIT"] = pre_audits["watchdog"] command = build_command(args, exporter, output) print("exec:", shlex.join(command), file=sys.stderr) - result = subprocess.run(command, env=environment, check=False) + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + result, executed_identity = run_approved_executable( + command, + path=exporter, + expected_path=candidate_policy["executable_path"], + expected_sha256=candidate_policy["executable_sha256"], + label="candidate exporter", + env=environment, + check=False, + ) + if executed_identity != exporter_identity: + raise PreflightError("candidate exporter execution identity differs from external approval") if result.returncode != 0: return result.returncode + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + post_runtime_build = query_runtime_build_attestation( + exporter, + args.device, + exporter_sha256=exporter_sha256, + candidate_revision=args.candidate_revision, + approval=candidate_policy, + ) + if post_runtime_build != pre_runtime_build: + raise PreflightError("candidate exporter build identity changed during trace execution") + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") verify_sealed_audits(pre_audits, pre_audit_digests) postflight_audit = run_strix_preflight( model=args.model, @@ -492,15 +689,21 @@ def main() -> int: repo=args.repo, busy_patterns=args.busy_pattern, ) - post_accelerator = query_accelerator_attestation(exporter, args.device) + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + post_accelerator = query_accelerator_attestation( + exporter, args.device, candidate_policy) if post_accelerator != accelerator: raise PreflightError("selected accelerator identity changed during trace execution") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") postflight_audit["runtime"] = "llama.cpp" postflight_audit["accelerator"] = post_accelerator post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) - bind_candidate_attestation(output, attestation, accelerator, exporter, exporter_sha256) + bind_candidate_attestation( + output, attestation, accelerator, exporter, exporter_sha256, candidate_policy) bind_execution_authorization(output, authorization) seal_bundle( output, @@ -509,6 +712,12 @@ def main() -> int: expected_lane=CANDIDATE_LANE, expected_challenge=args.execution_challenge, expected_run_id=args.run_id, + candidate_exporter_policies=approval_policy.candidate_exporters, + prompt_builder_policies=approval_policy.prompt_builders, + expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + expected_approval_policy_sha256=approval_policy.sha256, + expected_verifier_revision=approval_policy.verifier_revision, trusted_signers=APPROVED_TRACE_SIGNERS, ) bundle = TraceBundle( @@ -518,6 +727,10 @@ def main() -> int: expected_lane=CANDIDATE_LANE, expected_challenge=args.execution_challenge, expected_run_id=args.run_id, + expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, ), ) if bundle.manifest.get("runtime") != "llama.cpp": diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index e9a4c383417d..67dd0c05a6a9 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -12,6 +12,8 @@ from trace_format import ( ADMITTED_BATCH, ADMITTED_UBATCH, + APPROVED_CANDIDATE_EXPORTERS, + APPROVED_PROMPT_BUILDERS, APPROVED_TRACE_SIGNERS, CANDIDATE_LANE, CORPUS_SHA256, @@ -19,11 +21,21 @@ REQUIRED_EXPERT_CACHE_MIB, REQUIRED_EXPERT_SLOTS, TraceError, + approval_binding, + approved_executable_identity, + approved_prompt_record, + approved_runtime_file_identities, + candidate_exporter_approval, execution_authorization, + load_executable_approval_policy, + prompt_builder_approval, reject_loader_overrides, + run_approved_executable, sha256_file, strict_json_loads, validate_signing_identity, + verify_approved_executable_identity, + verify_approved_runtime_file_identities, ) CORPORA = ( @@ -46,16 +58,54 @@ def run(command: list[str]) -> None: raise RuntimeError(f"command failed with status {result.returncode}") +def file_identity(path: Path) -> tuple[int, int, int, int, int]: + record = path.stat() + return ( + record.st_dev, + record.st_ino, + record.st_size, + record.st_mtime_ns, + record.st_ctime_ns, + ) + + def prepare_prompt( *, builder: Path, + builder_approval_id: str, + builder_policy: dict[str, object], + builder_policy_sha256: str, model: Path, corpus: Path, + source_corpus: Path, corpus_name: str, corpus_sha256: str, output: Path, - target_tokens: int, + context: int, + decode_steps: int, ) -> dict[str, object]: + target_tokens = context - decode_steps + expected_prompt = approved_prompt_record( + builder_policy, + corpus_name=corpus_name, + context=context, + decode_steps=decode_steps, + ) + builder_identity = approved_executable_identity( + builder, + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + ) + runtime_identities = approved_runtime_file_identities( + builder_policy, label="prompt builder") + expected_source = Path(builder_policy["source_root"]) / "tests" / "corpus" / corpus_name + if source_corpus != expected_source or sha256_file(source_corpus) != corpus_sha256 or ( + sha256_file(corpus) != corpus_sha256): + raise RuntimeError("prompt builder corpus path or bytes differ from external approval") + source_identity = file_identity(source_corpus) + corpus_identity = file_identity(corpus) + verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") command = [ str(builder), "--model", str(model), @@ -64,7 +114,23 @@ def prepare_prompt( "--tokens", str(target_tokens), ] print("exec:", " ".join(command), file=sys.stderr) - result = subprocess.run(command, check=False, capture_output=True, text=True) + result, executed_identity = run_approved_executable( + command, + path=builder, + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + check=False, + capture_output=True, + text=True, + ) + if executed_identity != builder_identity: + raise RuntimeError("prompt builder execution identity differs from external approval") + verify_approved_executable_identity(builder, builder_identity, label="prompt builder") + verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") + if file_identity(source_corpus) != source_identity or file_identity(corpus) != corpus_identity or ( + sha256_file(source_corpus) != corpus_sha256) or sha256_file(corpus) != corpus_sha256: + raise RuntimeError("prompt builder corpus changed during execution") if result.returncode != 0: raise RuntimeError(f"prompt builder failed: {result.stderr.strip()}") try: @@ -84,15 +150,29 @@ def prepare_prompt( expected_temporary_directory = os.environ.get("TMPDIR") if not expected_temporary_directory or temporary_directory != str(resolved(Path(expected_temporary_directory))): raise RuntimeError("prompt builder did not attest the selected temporary directory") + prompt_sha256 = sha256_file(output) + prompt_byte_count = output.stat().st_size + if prompt_sha256 != expected_prompt["prompt_sha256"] or ( + prompt_byte_count != expected_prompt["prompt_byte_count"]) or ( + native_record["add_bos"] != expected_prompt["add_bos"]): + raise RuntimeError("prompt builder output differs from external approval") record = { "format": "dsv41-prompt-provenance", "version": 1, "corpus_name": corpus_name, "corpus_sha256": corpus_sha256, + "corpus_path": str(source_corpus), "model_sha256": MODEL_SHA256, - "prompt_sha256": sha256_file(output), - "prompt_byte_count": output.stat().st_size, - "builder_sha256": sha256_file(builder), + "prompt_sha256": prompt_sha256, + "prompt_byte_count": prompt_byte_count, + "context": context, + "decode_steps": decode_steps, + "builder_approval_id": builder_approval_id, + "builder_approval_sha256": builder_policy_sha256, + "builder_path": str(builder), + "builder_sha256": builder_identity.sha256, + "builder_revision": builder_policy["revision"], + "builder_runtime_profile": builder_policy["runtime_profile"], "target_tokens": target_tokens, "actual_tokens": target_tokens, } @@ -120,6 +200,11 @@ def main() -> int: parser.add_argument("--candidate-revision", required=True) parser.add_argument("--base-revision", required=True) parser.add_argument("--candidate-diff-sha256", required=True) + parser.add_argument("--candidate-exporter-policy-id", required=True) + parser.add_argument("--prompt-builder-policy-id", required=True) + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) parser.add_argument("--llama-only", action="store_true") parser.add_argument("--contexts", type=int, nargs="+", default=[32768]) parser.add_argument("--ubatches", type=int, nargs="+", default=[ADMITTED_UBATCH]) @@ -152,14 +237,52 @@ def main() -> int: raise PreflightError( f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_CACHE_MIB} MiB expert cache") reject_loader_overrides() + output_candidate = resolved(args.output) + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(output_candidate,), + ) + candidate_policy, candidate_policy_sha256 = candidate_exporter_approval( + args.candidate_exporter_policy_id, + policies=approval_policy.candidate_exporters, + ) + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + args.prompt_builder_policy_id, + policies=approval_policy.prompt_builders, + ) + approved_executable_identity( + args.llama_exporter, + expected_path=candidate_policy["executable_path"], + expected_sha256=candidate_policy["executable_sha256"], + label="candidate exporter", + ) + if args.candidate_revision != candidate_policy["revision"] or ( + args.base_revision != candidate_policy["base_revision"]) or ( + args.candidate_diff_sha256 != candidate_policy["diff_sha256"]): + raise PreflightError("matrix candidate identity differs from external exporter approval") execution_authorization( lane=CANDIDATE_LANE, challenge=args.execution_challenge, run_id=f"{args.run_id_prefix}-preflight", issued_unix=args.authorization_issued_unix, expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + approvals={ + "candidate_exporter": approval_binding( + "candidate_exporter", + args.candidate_exporter_policy_id, + candidate_policy_sha256, + ), + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + ), + }, ) - output_candidate = resolved(args.output) if not args.llama_only: raise PreflightError( "cross-runtime capture must run on separate Strix and Apple hosts; " @@ -171,6 +294,12 @@ def main() -> int: forbidden_root=output_candidate, ) repo = resolved(args.repo) + revision = subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + stderr=subprocess.STDOUT, + ).decode("ascii").strip() + if revision != approval_policy.verifier_revision: + raise PreflightError("matrix verifier checkout differs from the external approval policy") output = require_nvme_path(output_candidate, "matrix output") model = require_nvme_path(args.model, "model") if not model.is_file(): @@ -189,9 +318,15 @@ def main() -> int: repo=repo, busy_patterns=args.busy_pattern, ) - prompt_builder = resolved(args.llama_prompt_builder) - if not prompt_builder.is_file() or not os.access(prompt_builder, os.X_OK): - raise PreflightError(f"prompt builder is not executable: {prompt_builder}") + prompt_builder = args.llama_prompt_builder + approved_executable_identity( + prompt_builder, + expected_path=prompt_policy["executable_path"], + expected_sha256=prompt_policy["executable_sha256"], + label="prompt builder", + ) + if str(repo) != prompt_policy["source_root"] or args.candidate_revision != prompt_policy["revision"]: + raise PreflightError("matrix repository or revision differs from prompt builder approval") if output.exists() and any(output.iterdir()): raise PreflightError(f"matrix output directory is not empty: {output}") inputs = output / "inputs" @@ -239,12 +374,17 @@ def main() -> int: ) prepared = prepare_prompt( builder=resolved(args.llama_prompt_builder), + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, model=model, corpus=Path(corpus["path"]), + source_corpus=Path(corpus["source"]), corpus_name=corpus["name"], corpus_sha256=corpus["sha256"], output=prompt, - target_tokens=target_tokens, + context=context, + decode_steps=args.decode_steps, ) prepared.update({"corpus": corpus["name"], "context": context}) prepared_prompts[corpus["name"]] = prepared @@ -276,6 +416,11 @@ def main() -> int: "--candidate-revision", args.candidate_revision, "--base-revision", args.base_revision, "--candidate-diff-sha256", args.candidate_diff_sha256, + "--candidate-exporter-policy-id", args.candidate_exporter_policy_id, + "--prompt-builder-policy-id", args.prompt_builder_policy_id, + "--approval-policy", str(resolved(args.approval_policy)), + "--approval-signature", str(resolved(args.approval_signature)), + "--approval-principal", args.approval_principal, "--output", str(llama_output), "--batch", str(args.batch), "--ubatch", str(ubatch), diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 387e9f0d691f..67ba4a3c9435 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -21,6 +21,12 @@ DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" APPROVED_EXPORTERS: dict[str, str] = {} APPROVED_TRACE_SIGNERS: dict[str, dict[str, str]] = {} +APPROVED_EXECUTABLE_APPROVERS: dict[str, str] = {} +APPROVED_CANDIDATE_EXPORTERS: dict[str, dict[str, Any]] = {} +APPROVED_PROMPT_BUILDERS: dict[str, dict[str, Any]] = {} +EXECUTABLE_APPROVAL_FORMAT = "dsv41-executable-approval" +EXECUTABLE_APPROVAL_VERSION = 1 +EXECUTABLE_APPROVAL_NAMESPACE = "dsv41-executable-approval-v1" SEAL_FORMAT = "dsv41-trace-bundle-signature" SEAL_VERSION = 1 SEAL_NAMESPACE = "dsv41-trace-bundle-v1" @@ -136,6 +142,12 @@ class TraceVerifier: expected_challenge: str expected_run_id: str verification_unix: int + candidate_exporter_policies: dict[str, dict[str, Any]] + prompt_builder_policies: dict[str, dict[str, Any]] + expected_candidate_exporter_policy_id: str | None + expected_prompt_builder_policy_id: str + expected_approval_policy_sha256: str + expected_verifier_revision: str seen_run_ids: set[str] | None = None test_only: bool = False @@ -147,6 +159,9 @@ def production( expected_lane: str, expected_challenge: str, expected_run_id: str, + expected_candidate_exporter_policy_id: str | None, + expected_prompt_builder_policy_id: str, + approval_policy: "ExecutableApprovalPolicy | None" = None, verification_unix: int | None, seen_run_ids: set[str] | None = None) -> "TraceVerifier": return cls( @@ -157,6 +172,18 @@ def production( expected_challenge=expected_challenge, expected_run_id=expected_run_id, verification_unix=int(time.time()) if verification_unix is None else verification_unix, + candidate_exporter_policies=( + approval_policy.candidate_exporters + if approval_policy is not None else APPROVED_CANDIDATE_EXPORTERS), + prompt_builder_policies=( + approval_policy.prompt_builders + if approval_policy is not None else APPROVED_PROMPT_BUILDERS), + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=( + approval_policy.sha256 if approval_policy is not None else ""), + expected_verifier_revision=( + approval_policy.verifier_revision if approval_policy is not None else ""), seen_run_ids=seen_run_ids, ) @@ -171,6 +198,12 @@ def for_tests( runtime_profile: str, expected_challenge: str, expected_run_id: str, + candidate_exporter_policies: dict[str, dict[str, Any]] | None = None, + prompt_builder_policies: dict[str, dict[str, Any]] | None = None, + expected_candidate_exporter_policy_id: str | None = None, + expected_prompt_builder_policy_id: str = "", + expected_approval_policy_sha256: str = "e" * 64, + expected_verifier_revision: str = "a" * 40, verification_unix: int | None = None, ssh_keygen: Path | None = None, seen_run_ids: set[str] | None = None) -> "TraceVerifier": @@ -189,6 +222,12 @@ def for_tests( expected_challenge=expected_challenge, expected_run_id=expected_run_id, verification_unix=verification_unix, + candidate_exporter_policies=candidate_exporter_policies or {}, + prompt_builder_policies=prompt_builder_policies or {}, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=expected_approval_policy_sha256, + expected_verifier_revision=expected_verifier_revision, seen_run_ids=seen_run_ids, test_only=True, ) @@ -204,6 +243,26 @@ class BundleFileReceipt: sha256: str +@dataclass(frozen=True) +class ExecutableFileReceipt: + path: str + device: int + inode: int + byte_count: int + modified_ns: int + changed_ns: int + sha256: str + + +@dataclass(frozen=True) +class ExecutableApprovalPolicy: + principal: str + verifier_revision: str + candidate_exporters: dict[str, dict[str, Any]] + prompt_builders: dict[str, dict[str, Any]] + sha256: str + + @dataclass(frozen=True) class Mismatch: classification: str @@ -251,6 +310,184 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def _approved_file_identity( + path: Path, + *, + expected_path: str, + expected_sha256: str, + label: str, + executable: bool, +) -> tuple[ExecutableFileReceipt, int]: + if not path.is_absolute() or str(path) != expected_path: + raise TraceError(f"{label} path differs from external approval") + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + if current.is_symlink(): + raise TraceError(f"{label} path must not use symlinks") + flags = os.O_RDONLY + flags |= getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + except OSError as error: + raise TraceError(f"cannot open {label}: {error}") from error + if not stat.S_ISREG(before.st_mode) or (executable and not os.access(path, os.X_OK)): + os.close(descriptor) + raise TraceError(f"{label} is not an approved regular file") + try: + digest = hashlib.sha256() + with os.fdopen(os.dup(descriptor), "rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + digest_value = digest.hexdigest() + after = os.fstat(descriptor) + path_after = path.stat(follow_symlinks=False) + except OSError as error: + os.close(descriptor) + raise TraceError(f"cannot recheck {label}: {error}") from error + identity = ( + before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) + if identity != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) or identity != ( + path_after.st_dev, path_after.st_ino, path_after.st_size, + path_after.st_mtime_ns, path_after.st_ctime_ns): + os.close(descriptor) + raise TraceError(f"{label} changed while hashing") + if digest_value != expected_sha256: + os.close(descriptor) + raise TraceError(f"{label} SHA-256 differs from external approval") + return ExecutableFileReceipt( + path=str(path), + device=after.st_dev, + inode=after.st_ino, + byte_count=after.st_size, + modified_ns=after.st_mtime_ns, + changed_ns=after.st_ctime_ns, + sha256=digest_value, + ), descriptor + + +def approved_executable_identity( + path: Path, + *, + expected_path: str, + expected_sha256: str, + label: str, +) -> ExecutableFileReceipt: + identity, descriptor = _approved_file_identity( + path, + expected_path=expected_path, + expected_sha256=expected_sha256, + label=label, + executable=True, + ) + os.close(descriptor) + return identity + + +def verify_approved_executable_identity( + path: Path, + expected: ExecutableFileReceipt, + *, + label: str, +) -> None: + observed = approved_executable_identity( + path, + expected_path=expected.path, + expected_sha256=expected.sha256, + label=label, + ) + if observed != expected: + raise TraceError(f"{label} identity changed after approval") + + +def approved_runtime_file_identities( + policy: dict[str, Any], + *, + label: str, +) -> list[ExecutableFileReceipt]: + install_root = Path(policy["install_root"]) + identities = [] + for component in policy["runtime_receipt"]["components"]: + path = install_root / "lib" / component["filename"] + identity, descriptor = _approved_file_identity( + path, + expected_path=str(path), + expected_sha256=component["sha256"], + label=f"{label} runtime component {component['component']}", + executable=False, + ) + os.close(descriptor) + identities.append(identity) + return identities + + +def verify_approved_runtime_file_identities( + identities: list[ExecutableFileReceipt], + *, + label: str, +) -> None: + for identity in identities: + observed, descriptor = _approved_file_identity( + Path(identity.path), + expected_path=identity.path, + expected_sha256=identity.sha256, + label=f"{label} runtime component", + executable=False, + ) + os.close(descriptor) + if observed != identity: + raise TraceError(f"{label} runtime component identity changed after approval") + + +def run_approved_executable( + command: list[str], + *, + path: Path, + expected_path: str, + expected_sha256: str, + label: str, + **kwargs: Any, +) -> tuple[subprocess.CompletedProcess[Any], ExecutableFileReceipt]: + if sys.platform != "linux": + raise TraceError(f"{label} descriptor execution requires Linux") + identity, descriptor = _approved_file_identity( + path, + expected_path=expected_path, + expected_sha256=expected_sha256, + label=label, + executable=True, + ) + try: + result = subprocess.run( + command, + executable=f"/proc/self/fd/{descriptor}", + pass_fds=(descriptor,), + **kwargs, + ) + descriptor_after = os.fstat(descriptor) + if ( + descriptor_after.st_dev, + descriptor_after.st_ino, + descriptor_after.st_size, + descriptor_after.st_mtime_ns, + descriptor_after.st_ctime_ns, + ) != ( + identity.device, + identity.inode, + identity.byte_count, + identity.modified_ns, + identity.changed_ns, + ): + raise TraceError(f"{label} descriptor identity changed during execution") + verify_approved_executable_identity(path, identity, label=label) + return result, identity + finally: + os.close(descriptor) + + def reject_loader_overrides(environment: dict[str, str] | None = None) -> None: values = os.environ if environment is None else environment active = sorted(name for name in FORBIDDEN_LOADER_ENVIRONMENT if values.get(name)) @@ -357,6 +594,415 @@ def _normalize_public_key(public_key: str, *, allow_comment: bool = False) -> st return " ".join(fields[:2]) +def _approval_path(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.startswith("/") or ( + ".." in PurePosixPath(value).parts or str(PurePosixPath(value)) != value): + raise TraceError(f"{label} is not an absolute canonical path") + return value + + +def _approval_digest(kind: str, approval_id: str, policy: dict[str, Any]) -> str: + if re.fullmatch(r"[A-Za-z0-9._-]{1,128}", approval_id) is None: + raise TraceError(f"{kind} approval ID is invalid") + record = { + "format": EXECUTABLE_APPROVAL_FORMAT, + "version": EXECUTABLE_APPROVAL_VERSION, + "kind": kind, + "id": approval_id, + "policy": policy, + } + return sha256_bytes(canonical_json(record).encode("ascii")) + + +def candidate_exporter_approval( + approval_id: str, + *, + policies: dict[str, dict[str, Any]] = APPROVED_CANDIDATE_EXPORTERS, +) -> tuple[dict[str, Any], str]: + policy = policies.get(approval_id) + if not isinstance(policy, dict): + raise TraceError(f"candidate exporter approval is not trusted: {approval_id}") + _require_exact_keys( + policy, + { + "runtime", + "repository", + "revision", + "base_revision", + "diff_sha256", + "install_root", + "executable_path", + "executable_sha256", + "runtime_profile", + "runtime_receipt", + }, + "candidate exporter approval", + ) + if policy["runtime"] != "llama.cpp" or policy["repository"] != REPOSITORY: + raise TraceError("candidate exporter approval runtime identity is invalid") + for key in ("revision", "base_revision"): + if re.fullmatch(r"[0-9a-f]{40}", policy.get(key, "")) is None: + raise TraceError(f"candidate exporter approval {key} is invalid") + for key in ("diff_sha256", "executable_sha256"): + if re.fullmatch(r"[0-9a-f]{64}", policy.get(key, "")) is None: + raise TraceError(f"candidate exporter approval {key} is invalid") + install_root = _approval_path(policy["install_root"], "candidate exporter approval install root") + executable_path = _approval_path( + policy["executable_path"], "candidate exporter approval executable path") + if executable_path != f"{install_root}/bin/llama-deepseek-v41-trace": + raise TraceError("candidate exporter approval executable path is outside its install policy") + profile = policy["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("candidate exporter approval runtime profile is invalid") + _require_exact_keys( + profile, {"name", "components", "selected_backend_component"}, + "candidate exporter approval runtime profile") + if profile["name"] != "sibling-lib" or profile["selected_backend_component"] != "ggml-hip": + raise TraceError("candidate exporter approval runtime profile is invalid") + components = profile["components"] + if not isinstance(components, list) or components != sorted(components) or ( + len(components) != len(set(components))) or any( + not isinstance(component, str) or re.fullmatch(r"[a-z0-9-]+", component) is None + for component in components): + raise TraceError("candidate exporter approval components are invalid") + if not {"llama-common", "llama", "ggml", "ggml-base", "ggml-hip"}.issubset(set(components)): + raise TraceError("candidate exporter approval is missing required components") + receipt = policy["runtime_receipt"] + if not isinstance(receipt, dict): + raise TraceError("candidate exporter approval runtime receipt is invalid") + _require_exact_keys( + receipt, {"format", "version", "revision", "profile", "components"}, + "candidate exporter approval runtime receipt") + if receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or ( + receipt["revision"] != policy["revision"]) or receipt["profile"] != profile["name"]: + raise TraceError("candidate exporter approval runtime receipt identity is invalid") + receipt_components = receipt["components"] + if not isinstance(receipt_components, list) or receipt_components != sorted( + receipt_components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + raise TraceError("candidate exporter approval runtime receipt components are not canonical") + seen_components = set() + seen_filenames = set() + seen_digests = set() + for component in receipt_components: + if not isinstance(component, dict): + raise TraceError("candidate exporter approval runtime receipt component is invalid") + _require_exact_keys( + component, {"component", "filename", "sha256", "revision"}, + "candidate exporter approval runtime receipt component") + name = component["component"] + filename = component["filename"] + digest = component["sha256"] + revision = component["revision"] + if name not in components or name in seen_components: + raise TraceError("candidate exporter approval runtime receipt component name is invalid") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None or ( + filename in seen_filenames): + raise TraceError("candidate exporter approval runtime receipt filename is invalid") + if re.fullmatch(r"[0-9a-f]{64}", digest or "") is None or digest in seen_digests: + raise TraceError("candidate exporter approval runtime receipt digest is invalid") + revision_bearing = name in {"llama-common", "ggml-base"} + if (revision_bearing and revision != policy["revision"]) or ( + not revision_bearing and revision is not None): + raise TraceError("candidate exporter approval runtime receipt revision is invalid") + seen_components.add(name) + seen_filenames.add(filename) + seen_digests.add(digest) + if seen_components != set(components): + raise TraceError("candidate exporter approval receipt differs from its runtime profile") + return policy, _approval_digest("candidate-exporter", approval_id, policy) + + +def prompt_builder_approval( + approval_id: str, + *, + policies: dict[str, dict[str, Any]] = APPROVED_PROMPT_BUILDERS, +) -> tuple[dict[str, Any], str]: + policy = policies.get(approval_id) + if not isinstance(policy, dict): + raise TraceError(f"prompt builder approval is not trusted: {approval_id}") + _require_exact_keys( + policy, + { + "runtime", + "runtime_profile", + "repository", + "revision", + "install_root", + "executable_path", + "executable_sha256", + "source_root", + "runtime_receipt", + "model_sha256", + "corpora", + "prompts", + }, + "prompt builder approval", + ) + if policy["runtime"] != "llama.cpp" or ( + policy["repository"] != REPOSITORY) or re.fullmatch( + r"[0-9a-f]{40}", policy.get("revision", "")) is None: + raise TraceError("prompt builder approval runtime identity is invalid") + if policy["model_sha256"] != MODEL_SHA256 or re.fullmatch( + r"[0-9a-f]{64}", policy.get("executable_sha256", "")) is None: + raise TraceError("prompt builder approval executable or model identity is invalid") + install_root = _approval_path(policy["install_root"], "prompt builder approval install root") + executable_path = _approval_path( + policy["executable_path"], "prompt builder approval executable path") + source_root = _approval_path(policy["source_root"], "prompt builder approval source root") + if executable_path != f"{install_root}/bin/llama-deepseek-v41-prompt-builder": + raise TraceError("prompt builder approval executable path is outside its install policy") + profile = policy["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("prompt builder approval runtime profile is invalid") + _require_exact_keys( + profile, {"name", "components", "selected_backend_component"}, + "prompt builder approval runtime profile") + if profile["name"] != "sibling-lib" or not isinstance(profile["components"], list) or ( + profile["components"] != sorted(profile["components"])) or len( + profile["components"]) != len(set(profile["components"])) or not { + "llama-common", "llama", "ggml", "ggml-base" + }.issubset(set(profile["components"])) or ( + profile["selected_backend_component"] != "ggml-hip") or ( + profile["selected_backend_component"] not in profile["components"]): + raise TraceError("prompt builder approval runtime profile is invalid") + receipt = policy["runtime_receipt"] + if not isinstance(receipt, dict): + raise TraceError("prompt builder approval runtime receipt is invalid") + _require_exact_keys( + receipt, {"format", "version", "revision", "profile", "components"}, + "prompt builder approval runtime receipt") + if receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or ( + receipt["revision"] != policy["revision"]) or receipt["profile"] != profile["name"]: + raise TraceError("prompt builder approval runtime receipt identity is invalid") + receipt_components = receipt["components"] + if not isinstance(receipt_components, list) or receipt_components != sorted( + receipt_components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + raise TraceError("prompt builder approval runtime receipt components are not canonical") + seen_components = set() + seen_filenames = set() + seen_digests = set() + for component in receipt_components: + if not isinstance(component, dict): + raise TraceError("prompt builder approval runtime receipt component is invalid") + _require_exact_keys( + component, {"component", "filename", "sha256", "revision"}, + "prompt builder approval runtime receipt component") + name = component["component"] + filename = component["filename"] + digest = component["sha256"] + revision = component["revision"] + if name not in profile["components"] or name in seen_components: + raise TraceError("prompt builder approval runtime receipt component name is invalid") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None or ( + filename in seen_filenames): + raise TraceError("prompt builder approval runtime receipt filename is invalid") + if re.fullmatch(r"[0-9a-f]{64}", digest or "") is None or digest in seen_digests: + raise TraceError("prompt builder approval runtime receipt digest is invalid") + revision_bearing = name in {"llama-common", "ggml-base"} + if (revision_bearing and revision != policy["revision"]) or ( + not revision_bearing and revision is not None): + raise TraceError("prompt builder approval runtime receipt revision is invalid") + seen_components.add(name) + seen_filenames.add(filename) + seen_digests.add(digest) + if seen_components != set(profile["components"]): + raise TraceError("prompt builder approval receipt differs from its runtime profile") + if policy["corpora"] != CORPUS_SHA256: + raise TraceError("prompt builder approval corpus policy is invalid") + prompts = policy["prompts"] + if not isinstance(prompts, list) or not prompts: + raise TraceError("prompt builder approval prompt policy is empty") + previous_key = None + seen_keys = set() + for prompt in prompts: + if not isinstance(prompt, dict): + raise TraceError("prompt builder approval prompt record is invalid") + _require_exact_keys( + prompt, + { + "corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", + "prompt_sha256", "prompt_byte_count", "add_bos", + }, + "prompt builder approval prompt record", + ) + corpus_name = prompt["corpus_name"] + if corpus_name not in CORPUS_SHA256 or prompt["corpus_sha256"] != CORPUS_SHA256[corpus_name]: + raise TraceError("prompt builder approval prompt corpus identity is invalid") + context = prompt["context"] + decode_steps = prompt["decode_steps"] + target_tokens = prompt["target_tokens"] + if type(context) is not int or context < 2 or type(decode_steps) is not int or decode_steps < 1 or ( + target_tokens != context - decode_steps): + raise TraceError("prompt builder approval prompt configuration is invalid") + if re.fullmatch(r"[0-9a-f]{64}", prompt.get("prompt_sha256", "")) is None or ( + type(prompt.get("prompt_byte_count")) is not int or prompt["prompt_byte_count"] <= 0) or ( + type(prompt.get("add_bos")) is not bool): + raise TraceError("prompt builder approval prompt output identity is invalid") + key = (corpus_name, context, decode_steps) + if key in seen_keys or (previous_key is not None and key <= previous_key): + raise TraceError("prompt builder approval prompt records are duplicated or unsorted") + seen_keys.add(key) + previous_key = key + return policy, _approval_digest("prompt-builder", approval_id, policy) + + +def approved_prompt_record( + policy: dict[str, Any], + *, + corpus_name: str, + context: int, + decode_steps: int, +) -> dict[str, Any]: + matches = [ + prompt for prompt in policy["prompts"] + if prompt["corpus_name"] == corpus_name and + prompt["context"] == context and + prompt["decode_steps"] == decode_steps + ] + if len(matches) != 1: + raise TraceError("prompt builder approval does not contain the requested prompt configuration") + return matches[0] + + +def _read_external_regular_file(path: Path, label: str) -> bytes: + if not path.is_absolute() or str(path.resolve()) != str(path): + raise TraceError(f"{label} path must be absolute, canonical, and non-symlinked") + try: + before = path.stat(follow_symlinks=False) + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + opened = os.fstat(descriptor) + except OSError as error: + raise TraceError(f"cannot inspect {label}: {error}") from error + identity = ( + before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) + if identity != ( + opened.st_dev, opened.st_ino, opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) or ( + not stat.S_ISREG(opened.st_mode)) or opened.st_nlink != 1: + os.close(descriptor) + raise TraceError(f"{label} must be a regular file with one link") + try: + with os.fdopen(os.dup(descriptor), "rb") as stream: + data = stream.read() + after = os.fstat(descriptor) + path_after = path.stat(follow_symlinks=False) + except OSError as error: + os.close(descriptor) + raise TraceError(f"cannot read {label}: {error}") from error + if identity != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) or ( + identity != ( + path_after.st_dev, path_after.st_ino, path_after.st_size, + path_after.st_mtime_ns, path_after.st_ctime_ns)) or len(data) != before.st_size: + os.close(descriptor) + raise TraceError(f"{label} changed while reading") + os.close(descriptor) + return data + + +def load_executable_approval_policy( + policy_path: Path, + signature_path: Path, + *, + expected_principal: str, + trusted_approvers: dict[str, str] = APPROVED_EXECUTABLE_APPROVERS, + ssh_keygen: Path | None = None, + forbidden_roots: Iterable[Path] = (), +) -> ExecutableApprovalPolicy: + _validate_principal(expected_principal) + resolved_policy = policy_path.resolve() + resolved_signature = signature_path.resolve() + for forbidden_root in forbidden_roots: + root = forbidden_root.resolve() + if resolved_policy == root or root in resolved_policy.parents or ( + resolved_signature == root) or root in resolved_signature.parents: + raise TraceError("executable approval policy and signature must be outside protected output roots") + public_key = trusted_approvers.get(expected_principal) + if not isinstance(public_key, str): + raise TraceError(f"executable approval principal is not trusted: {expected_principal}") + approved_key = _normalize_public_key(public_key) + policy_bytes = _read_external_regular_file(policy_path, "executable approval policy") + signature_bytes = _read_external_regular_file(signature_path, "executable approval signature") + try: + policy = strict_json_loads(policy_bytes.decode("ascii")) + signature = signature_bytes.decode("ascii") + except UnicodeError as error: + raise TraceError("executable approval policy and signature must be ASCII") from error + if policy_bytes != _canonical_json_bytes(policy): + raise TraceError("executable approval policy is not canonical") + if not isinstance(policy, dict): + raise TraceError("executable approval policy must be a JSON object") + _require_exact_keys( + policy, + { + "format", "version", "principal", "verifier_repository", "verifier_revision", + "candidate_exporters", "prompt_builders", + }, + "executable approval policy", + ) + if policy["format"] != EXECUTABLE_APPROVAL_FORMAT or ( + policy["version"] != EXECUTABLE_APPROVAL_VERSION) or ( + policy["principal"] != expected_principal) or ( + policy["verifier_repository"] != REPOSITORY) or re.fullmatch( + r"[0-9a-f]{40}", policy.get("verifier_revision", "")) is None: + raise TraceError("executable approval policy identity is invalid") + candidate_exporters = policy["candidate_exporters"] + prompt_builders = policy["prompt_builders"] + if not isinstance(candidate_exporters, dict) or not isinstance(prompt_builders, dict): + raise TraceError("executable approval policy maps are invalid") + for approval_id in sorted(candidate_exporters): + candidate_exporter_approval(approval_id, policies=candidate_exporters) + for approval_id in sorted(prompt_builders): + prompt_builder_approval(approval_id, policies=prompt_builders) + if not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or not signature.endswith( + "-----END SSH SIGNATURE-----\n"): + raise TraceError("executable approval signature is invalid") + executable = _validate_ssh_keygen(ssh_keygen or trusted_ssh_keygen_path()) + with tempfile.TemporaryDirectory(prefix="dsv41-approval-verify-") as temp: + temporary = Path(temp) + allowed_signers = temporary / "allowed_signers" + signature_file = temporary / "signature" + allowed_signers.write_text(f"{expected_principal} {approved_key}\n", encoding="ascii") + signature_file.write_text(signature, encoding="ascii") + try: + result = subprocess.run( + [ + str(executable), + "-Y", "verify", + "-f", str(allowed_signers), + "-I", expected_principal, + "-n", EXECUTABLE_APPROVAL_NAMESPACE, + "-s", str(signature_file), + ], + input=policy_bytes, + check=False, + capture_output=True, + timeout=30, + env=_ssh_environment(), + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot verify executable approval signature: {error}") from error + if result.returncode != 0: + raise TraceError("executable approval signature verification failed") + return ExecutableApprovalPolicy( + principal=expected_principal, + verifier_revision=policy["verifier_revision"], + candidate_exporters=candidate_exporters, + prompt_builders=prompt_builders, + sha256=sha256_bytes(policy_bytes), + ) + + +def approval_binding(kind: str, approval_id: str, digest: str) -> dict[str, str]: + if kind not in {"candidate_exporter", "prompt_builder"} or re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", approval_id) is None or re.fullmatch( + r"[0-9a-f]{64}", digest) is None: + raise TraceError("execution approval binding is invalid") + return {"id": approval_id, "sha256": digest} + + def validate_execution_authorization( manifest: dict[str, Any], *, @@ -365,6 +1011,12 @@ def validate_execution_authorization( expected_challenge: str, expected_run_id: str, verification_unix: int, + candidate_exporter_policies: dict[str, dict[str, Any]], + prompt_builder_policies: dict[str, dict[str, Any]], + expected_candidate_exporter_policy_id: str | None, + expected_prompt_builder_policy_id: str, + expected_approval_policy_sha256: str, + expected_verifier_revision: str, seen_run_ids: set[str] | None = None) -> None: if expected_lane not in {CANDIDATE_LANE, ORACLE_LANE}: raise TraceError("externally expected execution lane is invalid") @@ -380,7 +1032,10 @@ def validate_execution_authorization( raise TraceError("manifest execution authorization is missing") _require_exact_keys( authorization, - {"format", "version", "lane", "challenge", "run_id", "issued_unix", "expires_unix"}, + { + "format", "version", "lane", "challenge", "run_id", "issued_unix", + "expires_unix", "approval_policy_sha256", "verifier_revision", "approvals", + }, "manifest execution authorization", ) if authorization.get("format") != AUTHORIZATION_FORMAT or ( @@ -392,6 +1047,12 @@ def validate_execution_authorization( raise TraceError("manifest execution challenge differs from the external challenge") if authorization.get("run_id") != expected_run_id: raise TraceError("manifest lane run ID differs from the external run ID") + if re.fullmatch(r"[0-9a-f]{64}", expected_approval_policy_sha256) is None or ( + authorization.get("approval_policy_sha256") != expected_approval_policy_sha256): + raise TraceError("manifest executable approval policy differs from the external policy") + if re.fullmatch(r"[0-9a-f]{40}", expected_verifier_revision) is None or ( + authorization.get("verifier_revision") != expected_verifier_revision): + raise TraceError("manifest verifier revision differs from the external approval policy") issued_unix = authorization.get("issued_unix") expires_unix = authorization.get("expires_unix") if type(issued_unix) is not int or type(expires_unix) is not int or ( @@ -410,6 +1071,29 @@ def validate_execution_authorization( ) if runtime_profile != policy["runtime_profile"]: raise TraceError("trace signer runtime profile does not match the signed manifest") + _prompt_policy, prompt_digest = prompt_builder_approval( + expected_prompt_builder_policy_id, policies=prompt_builder_policies) + approvals = authorization["approvals"] + required_approvals = {"prompt_builder"} + if expected_lane == CANDIDATE_LANE: + required_approvals.add("candidate_exporter") + if not isinstance(approvals, dict): + raise TraceError("manifest execution approval bindings are invalid") + _require_exact_keys(approvals, required_approvals, "manifest execution approval bindings") + prompt_binding = approvals["prompt_builder"] + if prompt_binding != approval_binding( + "prompt_builder", expected_prompt_builder_policy_id, prompt_digest): + raise TraceError("manifest prompt builder approval differs from external policy") + if expected_lane == CANDIDATE_LANE: + if expected_candidate_exporter_policy_id is None: + raise TraceError("external candidate exporter approval ID is required") + _candidate_policy, candidate_digest = candidate_exporter_approval( + expected_candidate_exporter_policy_id, policies=candidate_exporter_policies) + if approvals["candidate_exporter"] != approval_binding( + "candidate_exporter", expected_candidate_exporter_policy_id, candidate_digest): + raise TraceError("manifest candidate exporter approval differs from external policy") + elif expected_candidate_exporter_policy_id is not None: + raise TraceError("oracle verification must not specify a candidate exporter approval") if seen_run_ids is not None: if expected_run_id in seen_run_ids: raise TraceError("trace lane run ID was reused") @@ -422,17 +1106,35 @@ def execution_authorization( challenge: str, run_id: str, issued_unix: int, - expires_unix: int) -> dict[str, Any]: - runtime, profile = { - CANDIDATE_LANE: ("llama.cpp", "sibling-lib"), - ORACLE_LANE: ("ds4", "apple-metal"), - }.get(lane, (None, None)) - policy = { - "public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - "lane": lane, - "runtime": runtime, - "runtime_profile": profile, - } + expires_unix: int, + approval_policy_sha256: str, + verifier_revision: str, + approvals: dict[str, dict[str, str]]) -> dict[str, Any]: + if lane not in {CANDIDATE_LANE, ORACLE_LANE}: + raise TraceError("execution authorization lane is invalid") + if re.fullmatch(r"[0-9a-f]{64}", challenge) is None: + raise TraceError("execution authorization challenge is invalid") + run_prefix = "strix-llama-" if lane == CANDIDATE_LANE else "apple-ds4-" + if re.fullmatch(re.escape(run_prefix) + r"[A-Za-z0-9._-]{1,96}", run_id) is None: + raise TraceError("execution authorization run ID is invalid") + if type(issued_unix) is not int or type(expires_unix) is not int or ( + issued_unix <= 0 or expires_unix <= issued_unix or + expires_unix - issued_unix > MAX_AUTHORIZATION_LIFETIME_SECONDS): + raise TraceError("execution authorization validity window is invalid") + if re.fullmatch(r"[0-9a-f]{64}", approval_policy_sha256) is None or re.fullmatch( + r"[0-9a-f]{40}", verifier_revision) is None: + raise TraceError("execution authorization approval policy identity is invalid") + required_approvals = {"prompt_builder"} + if lane == CANDIDATE_LANE: + required_approvals.add("candidate_exporter") + if not isinstance(approvals, dict): + raise TraceError("execution authorization approvals are invalid") + _require_exact_keys(approvals, required_approvals, "execution authorization approvals") + for kind, binding in approvals.items(): + if not isinstance(binding, dict): + raise TraceError("execution authorization approval binding is invalid") + if binding != approval_binding(kind, binding.get("id", ""), binding.get("sha256", "")): + raise TraceError("execution authorization approval binding is invalid") authorization = { "format": AUTHORIZATION_FORMAT, "version": AUTHORIZATION_VERSION, @@ -441,21 +1143,10 @@ def execution_authorization( "run_id": run_id, "issued_unix": issued_unix, "expires_unix": expires_unix, + "approval_policy_sha256": approval_policy_sha256, + "verifier_revision": verifier_revision, + "approvals": approvals, } - manifest = { - "runtime": runtime, - "authorization": authorization, - "build": {"runtime_profile": {"name": profile}}, - "accelerator": {"runtime_kind": profile}, - } - validate_execution_authorization( - manifest, - policy=policy, - expected_lane=lane, - expected_challenge=challenge, - expected_run_id=run_id, - verification_unix=int(time.time()), - ) return authorization @@ -799,6 +1490,12 @@ def seal_bundle( expected_lane: str, expected_challenge: str, expected_run_id: str, + candidate_exporter_policies: dict[str, dict[str, Any]] = APPROVED_CANDIDATE_EXPORTERS, + prompt_builder_policies: dict[str, dict[str, Any]] = APPROVED_PROMPT_BUILDERS, + expected_candidate_exporter_policy_id: str | None = None, + expected_prompt_builder_policy_id: str = "", + expected_approval_policy_sha256: str = "", + expected_verifier_revision: str = "", verification_unix: int | None = None, trusted_signers: dict[str, dict[str, str]] = APPROVED_TRACE_SIGNERS, ssh_keygen: Path | None = None) -> str: @@ -822,6 +1519,12 @@ def seal_bundle( expected_challenge=expected_challenge, expected_run_id=expected_run_id, verification_unix=int(time.time()) if verification_unix is None else verification_unix, + candidate_exporter_policies=candidate_exporter_policies, + prompt_builder_policies=prompt_builder_policies, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=expected_approval_policy_sha256, + expected_verifier_revision=expected_verifier_revision, ) try: with tempfile.TemporaryDirectory(prefix="dsv41-trace-sign-") as signing_temp: @@ -938,6 +1641,12 @@ def verify_bundle_seal( expected_challenge=verifier.expected_challenge, expected_run_id=verifier.expected_run_id, verification_unix=verifier.verification_unix, + candidate_exporter_policies=verifier.candidate_exporter_policies, + prompt_builder_policies=verifier.prompt_builder_policies, + expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, + expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, + expected_verifier_revision=verifier.expected_verifier_revision, seen_run_ids=verifier.seen_run_ids, ) domain_sha256 = sha256_bytes(domain) @@ -1556,28 +2265,38 @@ def __init__( expected_lane: str | None = None, expected_challenge: str | None = None, expected_run_id: str | None = None, + expected_candidate_exporter_policy_id: str | None = None, + expected_prompt_builder_policy_id: str | None = None, verification_unix: int | None = None, seen_run_ids: set[str] | None = None): if root.is_symlink(): raise TraceError("trace root must not be a symlink") self.root = root.resolve() if verifier is None: - if None in (signer_principal, expected_lane, expected_challenge, expected_run_id): + if None in ( + signer_principal, expected_lane, expected_challenge, expected_run_id, + expected_prompt_builder_policy_id): raise TraceError( - "external signer, lane, challenge, and run ID expectations are required") + "external signer, lane, challenge, run ID, and prompt builder approval are required") + if expected_lane == CANDIDATE_LANE and expected_candidate_exporter_policy_id is None: + raise TraceError("external candidate exporter approval is required") verifier = TraceVerifier.production( signer_principal, expected_lane=expected_lane, expected_challenge=expected_challenge, expected_run_id=expected_run_id, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, verification_unix=verification_unix, seen_run_ids=seen_run_ids, ) elif any(value is not None for value in ( signer_principal, expected_lane, expected_challenge, expected_run_id, + expected_candidate_exporter_policy_id, expected_prompt_builder_policy_id, verification_unix, seen_run_ids)): raise TraceError("trace verifier cannot be combined with separate verification inputs") self.signer_principal = verifier.principal + self.verifier = verifier ( self.manifest, sealed_events, @@ -1907,25 +2626,51 @@ def _validate_manifest(self) -> None: if sha256_bytes(provenance_bytes) != provenance_sha256: raise TraceError("prompt provenance SHA-256 mismatch") expected_target = context - decode_steps + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + self.verifier.expected_prompt_builder_policy_id, + policies=self.verifier.prompt_builder_policies, + ) provenance_checks = { "format": "dsv41-prompt-provenance", "version": 1, "corpus_name": corpus_name, "corpus_sha256": self.manifest["prompt"]["corpus_sha256"], + "corpus_path": f"{prompt_policy['source_root']}/tests/corpus/{corpus_name}", "model_sha256": self.manifest["model"]["sha256"], "prompt_sha256": self.manifest["prompt"]["sha256"], "prompt_byte_count": self.manifest["prompt"]["byte_count"], + "context": context, + "decode_steps": decode_steps, "target_tokens": expected_target, "actual_tokens": expected_target, } for key, value in provenance_checks.items(): if provenance_record.get(key) != value: raise TraceError(f"prompt provenance {key} mismatch") - if re.fullmatch(r"[0-9a-f]{64}", provenance_record.get("builder_sha256", "")) is None: - raise TraceError("prompt provenance builder SHA-256 is invalid") + expected_prompt = approved_prompt_record( + prompt_policy, + corpus_name=corpus_name, + context=context, + decode_steps=decode_steps, + ) + builder_checks = { + "builder_approval_id": self.verifier.expected_prompt_builder_policy_id, + "builder_approval_sha256": prompt_policy_sha256, + "builder_path": prompt_policy["executable_path"], + "builder_sha256": prompt_policy["executable_sha256"], + "builder_revision": prompt_policy["revision"], + "builder_runtime_profile": prompt_policy["runtime_profile"], + } + for key, value in builder_checks.items(): + if provenance_record.get(key) != value: + raise TraceError(f"prompt provenance {key} differs from external approval") + for key in ("corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", + "prompt_sha256", "prompt_byte_count"): + if provenance_record[key] != expected_prompt[key]: + raise TraceError(f"prompt provenance {key} differs from approved prompt output") _require_exact_keys( provenance_record, - set(provenance_checks) | {"builder_sha256"}, + set(provenance_checks) | set(builder_checks), "prompt provenance", ) if self.manifest["prompt"].get("target_tokens") != expected_target: @@ -1945,6 +2690,8 @@ def _validate_manifest(self) -> None: "executable_sha256", "runtime_libraries_sha256", "runtime_receipt_sha256", + "exporter_approval_id", + "exporter_approval_sha256", }, "llama.cpp candidate attestation", ) @@ -1956,11 +2703,15 @@ def _validate_manifest(self) -> None: "diff_sha256", "executable_sha256", "runtime_libraries_sha256", - "runtime_receipt_sha256"): + "runtime_receipt_sha256", + "exporter_approval_sha256"): value = candidate.get(key, "") if not isinstance(value, str) or re.fullmatch( r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: raise TraceError(f"candidate {key} is invalid") + if re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", candidate.get("exporter_approval_id", "")) is None: + raise TraceError("candidate exporter approval ID is invalid") executable_path = candidate.get("executable_path") if not isinstance(executable_path, str) or not executable_path.startswith("/") or ( ".." in PurePosixPath(executable_path).parts or @@ -1981,6 +2732,26 @@ def _validate_manifest(self) -> None: raise TraceError("candidate runtime library identities do not match the trace build") if candidate["runtime_receipt_sha256"] != self.manifest["build"]["runtime_receipt_sha256"]: raise TraceError("candidate runtime receipt does not match the trace build") + candidate_policy, candidate_policy_sha256 = candidate_exporter_approval( + self.verifier.expected_candidate_exporter_policy_id or "", + policies=self.verifier.candidate_exporter_policies, + ) + if candidate["exporter_approval_id"] != self.verifier.expected_candidate_exporter_policy_id or ( + candidate["exporter_approval_sha256"] != candidate_policy_sha256): + raise TraceError("candidate exporter approval differs from external policy") + policy_checks = { + "repository": candidate["repository"], + "revision": candidate["revision"], + "base_revision": candidate["base_revision"], + "diff_sha256": candidate["diff_sha256"], + "executable_path": candidate["executable_path"], + "executable_sha256": candidate["executable_sha256"], + "runtime_profile": self.manifest["build"]["runtime_profile"], + "runtime_receipt": receipt, + } + for key, value in policy_checks.items(): + if candidate_policy[key] != value: + raise TraceError(f"candidate {key} differs from external exporter approval") expected_config = { "layer_count": 40, "vocab_size": 129280, @@ -2799,13 +3570,28 @@ def report( def command_validate(args: argparse.Namespace) -> int: - bundle = TraceBundle( - args.bundle, - signer_principal=getattr(args, "signer_principal", None), - expected_lane=getattr(args, "lane", None), - expected_challenge=getattr(args, "execution_challenge", None), - expected_run_id=getattr(args, "run_id", None), - ) + if hasattr(args, "approval_policy"): + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(args.bundle,), + ) + bundle = TraceBundle( + args.bundle, + verifier=TraceVerifier.production( + args.signer_principal, + expected_lane=args.lane, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + ), + ) + else: + bundle = TraceBundle(args.bundle) print(canonical_json({ "status": "valid", "runtime": bundle.manifest.get("runtime"), @@ -2820,24 +3606,70 @@ def command_compare(args: argparse.Namespace) -> int: challenge = getattr(args, "execution_challenge", None) left_run_id = getattr(args, "left_run_id", None) right_run_id = getattr(args, "right_run_id", None) + approval_policy = ( + load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(args.left, args.right), + ) + if hasattr(args, "approval_policy") else None + ) seen_run_ids: set[str] = set() - result = report( - TraceBundle( + if approval_policy is None: + left_bundle = TraceBundle( args.left, signer_principal=left_principal, expected_lane=ORACLE_LANE, expected_challenge=challenge, expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=None, + expected_prompt_builder_policy_id=getattr(args, "prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, - ), - TraceBundle( + ) + right_bundle = TraceBundle( args.right, signer_principal=right_principal, expected_lane=CANDIDATE_LANE, expected_challenge=challenge, expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=getattr( + args, "right_candidate_exporter_policy_id", None), + expected_prompt_builder_policy_id=getattr(args, "prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, - ), + ) + else: + left_bundle = TraceBundle( + args.left, + verifier=TraceVerifier.production( + left_principal, + expected_lane=ORACLE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + right_bundle = TraceBundle( + args.right, + verifier=TraceVerifier.production( + right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=args.right_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + result = report( + left_bundle, + right_bundle, ) text = canonical_json(result) + "\n" if args.report: @@ -2898,24 +3730,73 @@ def command_compare_local(args: argparse.Namespace) -> int: challenge = getattr(args, "execution_challenge", None) left_run_id = getattr(args, "left_run_id", None) right_run_id = getattr(args, "right_run_id", None) + approval_policy = ( + load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(args.left, args.right), + ) + if hasattr(args, "approval_policy") else None + ) seen_run_ids: set[str] = set() - result = local_report( - TraceBundle( + if approval_policy is None: + left_bundle = TraceBundle( args.left, signer_principal=left_principal, expected_lane=CANDIDATE_LANE, expected_challenge=challenge, expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=getattr( + args, "left_candidate_exporter_policy_id", None), + expected_prompt_builder_policy_id=getattr( + args, "left_prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, - ), - TraceBundle( + ) + right_bundle = TraceBundle( args.right, signer_principal=right_principal, expected_lane=CANDIDATE_LANE, expected_challenge=challenge, expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=getattr( + args, "right_candidate_exporter_policy_id", None), + expected_prompt_builder_policy_id=getattr( + args, "right_prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, - ), + ) + else: + left_bundle = TraceBundle( + args.left, + verifier=TraceVerifier.production( + left_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=args.left_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=args.left_prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + right_bundle = TraceBundle( + args.right, + verifier=TraceVerifier.production( + right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=args.right_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=args.right_prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + result = local_report( + left_bundle, + right_bundle, args.mode, ) text = canonical_json(result) + "\n" @@ -2925,6 +3806,12 @@ def command_compare_local(args: argparse.Namespace) -> int: return 0 if result["status"] == "BRINGUP PASS" else 1 +def add_executable_approval_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Validate and compare DeepSeek V4.1 correctness traces") subparsers = parser.add_subparsers(dest="command", required=True) @@ -2934,6 +3821,9 @@ def build_parser() -> argparse.ArgumentParser: validate_parser.add_argument("--lane", choices=(CANDIDATE_LANE, ORACLE_LANE), required=True) validate_parser.add_argument("--execution-challenge", required=True) validate_parser.add_argument("--run-id", required=True) + validate_parser.add_argument("--candidate-exporter-policy-id") + validate_parser.add_argument("--prompt-builder-policy-id", required=True) + add_executable_approval_arguments(validate_parser) validate_parser.set_defaults(func=command_validate) compare_parser = subparsers.add_parser("compare") compare_parser.add_argument("left", type=Path) @@ -2943,6 +3833,9 @@ def build_parser() -> argparse.ArgumentParser: compare_parser.add_argument("--execution-challenge", required=True) compare_parser.add_argument("--left-run-id", required=True) compare_parser.add_argument("--right-run-id", required=True) + compare_parser.add_argument("--right-candidate-exporter-policy-id", required=True) + compare_parser.add_argument("--prompt-builder-policy-id", required=True) + add_executable_approval_arguments(compare_parser) compare_parser.add_argument("--report", type=Path) compare_parser.set_defaults(func=command_compare) local_parser = subparsers.add_parser("compare-local") @@ -2954,6 +3847,11 @@ def build_parser() -> argparse.ArgumentParser: local_parser.add_argument("--execution-challenge", required=True) local_parser.add_argument("--left-run-id", required=True) local_parser.add_argument("--right-run-id", required=True) + local_parser.add_argument("--left-candidate-exporter-policy-id", required=True) + local_parser.add_argument("--right-candidate-exporter-policy-id", required=True) + local_parser.add_argument("--left-prompt-builder-policy-id", required=True) + local_parser.add_argument("--right-prompt-builder-policy-id", required=True) + add_executable_approval_arguments(local_parser) local_parser.add_argument("--report", type=Path) local_parser.set_defaults(func=command_compare_local) return parser From c5c665f8c3e495197b101060e4ab3a106a27bb17 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 12:43:43 -0700 Subject: [PATCH 36/56] tools: bind trace tokenizer and loader trust Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 591 +++++++++++++++-- tools/deepseek-v41-trace/CMakeLists.txt | 13 +- tools/deepseek-v41-trace/README.md | 10 +- tools/deepseek-v41-trace/llama-trace.cpp | 28 +- tools/deepseek-v41-trace/preflight.py | 32 +- tools/deepseek-v41-trace/prompt-builder.cpp | 341 +++++++++- tools/deepseek-v41-trace/run_ds4.py | 57 +- tools/deepseek-v41-trace/run_llama.py | 100 +-- tools/deepseek-v41-trace/run_matrix.py | 91 ++- tools/deepseek-v41-trace/trace_format.py | 680 ++++++++++++++++++-- 10 files changed, 1768 insertions(+), 175 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 979d28bc5477..27b3d6394bbd 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import contextlib import copy import importlib.util import io @@ -391,6 +392,7 @@ def fixture_prompt_builder_policy( "repository": trace.REPOSITORY, "revision": "a" * 40, "install_root": str(Path(builder_path).parent.parent), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, "executable_path": builder_path, "executable_sha256": builder_sha256, "source_root": source_root, @@ -411,6 +413,13 @@ def fixture_prompt_builder_policy( }, "model_sha256": trace.MODEL_SHA256, "corpora": dict(trace.CORPUS_SHA256), + "tokenizer": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": True, + }, "prompts": [{ "corpus_name": "correctness-prose.txt", "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], @@ -419,7 +428,6 @@ def fixture_prompt_builder_policy( "target_tokens": context - decode_steps, "prompt_sha256": trace.sha256_bytes(prompt), "prompt_byte_count": len(prompt), - "add_bos": True, }], } @@ -431,6 +439,116 @@ def materialize_policy_runtime(policy: dict[str, object]) -> None: path = library_root / component["filename"] path.write_bytes(component["component"].encode("ascii")) component["sha256"] = trace.sha256_file(path) + path.chmod(0o555) + executable = Path(policy["executable_path"]) + if executable.exists(): + executable.chmod(0o555) + library_root.chmod(0o555) + executable.parent.chmod(0o555) + Path(policy["install_root"]).chmod(0o555) + + +def fixture_install_trust(policy: dict[str, object]) -> dict[str, object]: + install_root = Path(policy["install_root"]) + paths = { + policy["executable_path"]: policy["executable_sha256"], + **{ + str(install_root / "lib" / component["filename"]): component["sha256"] + for component in policy["runtime_receipt"]["components"] + }, + } + directory_paths = set() + for path in paths: + current = Path(path).parent + while True: + directory_paths.add(str(current)) + if current == Path(current.anchor): + break + current = current.parent + owner_uid = policy["install_owner_uid"] + directories = [] + for index, path in enumerate(sorted(directory_paths), 1): + in_install = path == str(install_root) or install_root in Path(path).parents + directories.append({ + "path": path, + "device": 1, + "inode": index, + "owner_uid": owner_uid if in_install else 0, + "mode": 0o555, + "effective_write_access": False, + "acl_entries": False, + }) + files = [] + for index, (path, digest) in enumerate(sorted(paths.items()), 100): + files.append({ + "path": path, + "device": 1, + "inode": index, + "owner_uid": owner_uid, + "mode": 0o555, + "link_count": 1, + "byte_count": index, + "modified_ns": index, + "changed_ns": index, + "sha256": digest, + "effective_write_access": False, + "acl_entries": False, + }) + return { + "format": "dsv41-install-trust", + "version": 1, + "install_root": str(install_root), + "owner_uid": owner_uid, + "execution_uid": owner_uid + 1 if owner_uid != 0 else 1, + "directories": directories, + "files": files, + } + + +def fixture_runtime_build(policy: dict[str, object]) -> dict[str, object]: + libraries = [] + for component in policy["runtime_receipt"]["components"]: + name = component["component"] + libraries.append({ + "component": name, + "filename": component["filename"], + "path": f"{policy['install_root']}/lib/{component['filename']}", + "sha256": component["sha256"], + "role": { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + }.get(name, f"runtime:{name}"), + "revision": component["revision"], + }) + libraries.sort(key=lambda item: item["path"]) + return { + "revision": policy["revision"], + "path": policy["executable_path"], + "sha256": policy["executable_sha256"], + "runtime_profile": policy["runtime_profile"], + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(policy["runtime_receipt"]).encode("ascii")), + "runtime_libraries": libraries, + "runtime_libraries_post": copy.deepcopy(libraries), + } + + +@contextlib.contextmanager +def isolated_test_install_trust(): + modules = (trace, sys.modules["trace_format"]) + with contextlib.ExitStack() as stack: + for module in modules: + stack.enter_context(mock.patch.object( + module, + "_execution_uid", + return_value=(os.geteuid() if hasattr(os, "geteuid") else 0) + 1, + )) + stack.enter_context(mock.patch.object( + module, "_path_is_writable_by_execution_identity", return_value=False)) + stack.enter_context(mock.patch.object( + module, "_has_access_control_entries", return_value=False)) + yield def provenance_bytes( @@ -443,6 +561,8 @@ def provenance_bytes( TEST_PROMPT_BUILDER_POLICY_ID, policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, ) + trust = fixture_install_trust(policy) + runtime_build = fixture_runtime_build(policy) record = { "format": "dsv41-prompt-provenance", "version": 1, @@ -462,6 +582,12 @@ def provenance_bytes( "builder_sha256": "8" * 64, "builder_revision": "a" * 40, "builder_runtime_profile": policy["runtime_profile"], + "tokenizer": policy["tokenizer"], + "builder_runtime_build": runtime_build, + "builder_runtime_build_sha256": trace.runtime_build_evidence_sha256( + runtime_build, policy, label="prompt builder"), + "builder_install_trust": trust, + "builder_install_trust_sha256": trace.install_trust_sha256(trust), } return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") @@ -643,8 +769,13 @@ def manifest( "device_pci_id": "0000:c1:00.0", "gpu_layers": 99, "load_mode": 0, - "tokenizer_add_bos": True, - "tokenizer_parse_special": True, + "tokenizer": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": True, + }, }) result["candidate"] = { "repository": trace.REPOSITORY, @@ -671,9 +802,14 @@ def manifest( TEST_PROMPT_BUILDER_POLICY_ID, policies={TEST_PROMPT_BUILDER_POLICY_ID: prompt_policy}, ) + prompt_trust = fixture_install_trust(prompt_policy) approvals = { "prompt_builder": trace.approval_binding( - "prompt_builder", TEST_PROMPT_BUILDER_POLICY_ID, prompt_policy_sha256), + "prompt_builder", + TEST_PROMPT_BUILDER_POLICY_ID, + prompt_policy_sha256, + trace.install_trust_sha256(prompt_trust), + ), } if not is_ds4: candidate_policy = { @@ -683,6 +819,7 @@ def manifest( "base_revision": result["candidate"]["base_revision"], "diff_sha256": result["candidate"]["diff_sha256"], "install_root": "/home/repo/build", + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, "executable_path": result["candidate"]["executable_path"], "executable_sha256": result["candidate"]["executable_sha256"], "runtime_profile": copy.deepcopy(result["build"]["runtime_profile"]), @@ -694,8 +831,15 @@ def manifest( ) result["candidate"]["exporter_approval_id"] = TEST_CANDIDATE_EXPORTER_POLICY_ID result["candidate"]["exporter_approval_sha256"] = candidate_policy_sha256 + candidate_trust = fixture_install_trust(candidate_policy) + result["candidate"]["install_trust"] = candidate_trust + result["candidate"]["install_trust_sha256"] = trace.install_trust_sha256(candidate_trust) approvals["candidate_exporter"] = trace.approval_binding( - "candidate_exporter", TEST_CANDIDATE_EXPORTER_POLICY_ID, candidate_policy_sha256) + "candidate_exporter", + TEST_CANDIDATE_EXPORTER_POLICY_ID, + candidate_policy_sha256, + trace.install_trust_sha256(candidate_trust), + ) result["authorization"] = trace.execution_authorization( lane=trace.ORACLE_LANE if is_ds4 else trace.CANDIDATE_LANE, challenge=TEST_CHALLENGE, @@ -704,6 +848,7 @@ def manifest( expires_unix=TEST_AUTH_EXPIRES, approval_policy_sha256="e" * 64, verifier_revision="a" * 40, + tokenizer_policy_sha256_value=trace.tokenizer_policy_sha256(prompt_policy["tokenizer"]), approvals=approvals, ) return result @@ -1065,6 +1210,7 @@ def _verifier_for_runtime( "base_revision": manifest_record["candidate"]["base_revision"], "diff_sha256": manifest_record["candidate"]["diff_sha256"], "install_root": "/home/repo/build", + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, "executable_path": manifest_record["candidate"]["executable_path"], "executable_sha256": manifest_record["candidate"]["executable_sha256"], "runtime_profile": copy.deepcopy(manifest_record["build"]["runtime_profile"]), @@ -1352,12 +1498,20 @@ def test_external_executable_approval_signature_and_tamper(self) -> None: capture_output=True, ) signature_path = policy_path.with_suffix(".json.sig") + approvers = { + principal: { + "public_key": public_key, + "policy_root": str(root), + "owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + }, + } loaded = trace.load_executable_approval_policy( policy_path, signature_path, expected_principal=principal, - trusted_approvers={principal: public_key}, + trusted_approvers=approvers, ssh_keygen=self.ssh_keygen, + test_only_trust=True, ) self.assertEqual(loaded.verifier_revision, "a" * 40) self.assertEqual(loaded.sha256, trace.sha256_file(policy_path)) @@ -1370,9 +1524,10 @@ def test_external_executable_approval_signature_and_tamper(self) -> None: policy_path, signature_path, expected_principal=principal, - trusted_approvers={principal: public_key}, + trusted_approvers=approvers, ssh_keygen=self.ssh_keygen, forbidden_roots=(root,), + test_only_trust=True, ) tampered = copy.deepcopy(policy) tampered["verifier_revision"] = "b" * 40 @@ -1382,8 +1537,9 @@ def test_external_executable_approval_signature_and_tamper(self) -> None: policy_path, signature_path, expected_principal=principal, - trusted_approvers={principal: public_key}, + trusted_approvers=approvers, ssh_keygen=self.ssh_keygen, + test_only_trust=True, ) with self.assertRaisesRegex(trace.TraceError, "principal is not trusted"): trace.load_executable_approval_policy( @@ -1392,8 +1548,73 @@ def test_external_executable_approval_signature_and_tamper(self) -> None: expected_principal=principal, trusted_approvers={}, ssh_keygen=self.ssh_keygen, + test_only_trust=True, ) + def test_external_approval_rejects_mutable_root_and_hardlinks(self) -> None: + verifier = self._verifier_for_runtime("llama.cpp") + principal = "dsv41-test-executable-approver" + policy = { + "format": trace.EXECUTABLE_APPROVAL_FORMAT, + "version": trace.EXECUTABLE_APPROVAL_VERSION, + "principal": principal, + "verifier_repository": trace.REPOSITORY, + "verifier_revision": "a" * 40, + "candidate_exporters": verifier.candidate_exporter_policies, + "prompt_builders": verifier.prompt_builder_policies, + } + for mutation, message in ( + ("mutable-root", "path is mutable"), + ("hard-linked-policy", "one-link regular file"), + ("hard-linked-signature", "one-link regular file"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy_path = root / "approval.json" + policy_path.write_text(trace.canonical_json(policy) + "\n", encoding="ascii") + subprocess.run( + [ + str(self.ssh_keygen), + "-Y", "sign", + "-f", str(self.signing_key), + "-n", trace.EXECUTABLE_APPROVAL_NAMESPACE, + str(policy_path), + ], + check=True, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + signature_path = policy_path.with_suffix(".json.sig") + policy_path.chmod(0o444) + signature_path.chmod(0o444) + root.chmod(0o555) + if mutation == "mutable-root": + root.chmod(0o777) + elif mutation == "hard-linked-policy": + os.link(policy_path, Path(temp).parent / f"{root.name}-policy-alias") + else: + os.link(signature_path, Path(temp).parent / f"{root.name}-signature-alias") + approvers = { + principal: { + "public_key": self.test_signers[self.signer_principal]["public_key"], + "policy_root": str(root), + "owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + }, + } + try: + with isolated_test_install_trust(), self.assertRaisesRegex(trace.TraceError, message): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers=approvers, + ssh_keygen=self.ssh_keygen, + ) + finally: + root.chmod(0o755) + for alias in Path(temp).parent.glob(f"{root.name}-*-alias"): + alias.unlink() + def test_candidate_runner_rejects_unapproved_exporter_before_execution(self) -> None: argv = [ "run_llama.py", @@ -1468,15 +1689,34 @@ def test_prompt_builder_rejects_unapproved_identity_before_execution(self) -> No def test_approved_executable_uses_linux_descriptor_path(self) -> None: with tempfile.TemporaryDirectory() as temp: - executable = Path(temp).resolve() / "approved" + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + library = install / "lib" / "libapproved.so" + executable.parent.mkdir(parents=True) + library.parent.mkdir() executable.write_bytes(b"approved") - executable.chmod(0o755) + library.write_bytes(b"approved library") + executable.chmod(0o555) + library.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": { + "components": [{ + "component": "approved", + "filename": library.name, + "sha256": trace.sha256_file(library), + }], + }, + } completed = subprocess.CompletedProcess([str(executable)], 0, "", "") - with mock.patch.object(trace.sys, "platform", "linux"), mock.patch.object( + with isolated_test_install_trust(), mock.patch.object( + trace.sys, "platform", "linux"), mock.patch.object( trace.subprocess, "run", return_value=completed) as execute: result, identity = trace.run_approved_executable( [str(executable), "--version"], path=executable, + runtime_policy=policy, expected_path=str(executable), expected_sha256=trace.sha256_file(executable), label="approved executable", @@ -1488,7 +1728,112 @@ def test_approved_executable_uses_linux_descriptor_path(self) -> None: self.assertEqual(identity.path, str(executable)) kwargs = execute.call_args.kwargs self.assertRegex(kwargs["executable"], r"^/proc/self/fd/[0-9]+$") - self.assertEqual(len(kwargs["pass_fds"]), 1) + self.assertEqual(execute.call_args.args[0][0], str(executable)) + self.assertEqual(len(kwargs["pass_fds"]), 2) + + def test_approved_install_rejects_mutable_alias_and_hardlink_paths(self) -> None: + for mutation, message in ( + ("writable-root", "path is mutable"), + ("writable-ancestor", "path is mutable"), + ("writable-executable", "immutable trusted-owned"), + ("hard-linked-library", "immutable trusted-owned"), + ("symlinked-executable", "canonical|aliases"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + install = root / "install" + executable = install / "bin" / "approved" + library = install / "lib" / "libapproved.so" + executable.parent.mkdir(parents=True) + library.parent.mkdir() + executable.write_bytes(b"approved executable") + library.write_bytes(b"approved library") + alias = install / "bin" / "alias" + if mutation == "symlinked-executable": + alias.symlink_to(executable) + executable.chmod(0o555) + library.chmod(0o555) + executable.parent.chmod(0o555) + library.parent.chmod(0o555) + install.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": { + "components": [{ + "component": "approved", + "filename": library.name, + "sha256": trace.sha256_file(library), + }], + }, + } + path = executable + if mutation == "writable-root": + install.chmod(0o777) + elif mutation == "writable-ancestor": + root.chmod(0o777) + elif mutation == "writable-executable": + executable.chmod(0o755) + elif mutation == "hard-linked-library": + os.link(library, root / "library-alias") + else: + path = alias + with isolated_test_install_trust(), self.assertRaisesRegex(trace.TraceError, message): + if mutation == "hard-linked-library": + trace.approved_runtime_file_identities(policy, label="approved") + else: + trace.approved_executable_identity( + path, + install_root=str(install), + expected_owner_uid=policy["install_owner_uid"], + expected_path=str(path), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + ) + + def test_writable_install_root_blocks_replace_restore_before_launch(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"approved executable") + executable.chmod(0o555) + install.chmod(0o777) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": {"components": []}, + } + with isolated_test_install_trust(), mock.patch.object( + trace.sys, "platform", "linux"), mock.patch.object( + trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace.TraceError, "path is mutable"): + trace.run_approved_executable( + [str(executable)], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + ) + execute.assert_not_called() + + def test_install_trust_evidence_rejects_mutability_claims(self) -> None: + policy = fixture_prompt_builder_policy(b"prompt") + mutations = { + "same-owner": lambda value: value.update({"execution_uid": value["owner_uid"]}), + "writable-directory": lambda value: value["directories"][-1].update({"mode": 0o777}), + "directory-acl": lambda value: value["directories"][-1].update({"acl_entries": True}), + "writable-file": lambda value: value["files"][0].update({"mode": 0o755}), + "hard-linked-file": lambda value: value["files"][0].update({"link_count": 2}), + "file-acl": lambda value: value["files"][0].update({"acl_entries": True}), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + trust = fixture_install_trust(policy) + mutate(trust) + with self.assertRaisesRegex(trace.TraceError, "install trust"): + trace.validate_install_trust_evidence(trust, policy) def test_prompt_builder_rejects_runtime_receipt_before_execution(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -1509,10 +1854,12 @@ def test_prompt_builder_rejects_runtime_receipt_before_execution(self) -> None: policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, ) runtime_component = policy["runtime_receipt"]["components"][0] - (Path(policy["install_root"]) / "lib" / runtime_component["filename"]).write_bytes( - b"changed") - with mock.patch.object(run_matrix, "run_approved_executable") as execute, self.assertRaisesRegex( - run_matrix.TraceError, "runtime component .* SHA-256 differs from external approval"): + runtime_path = Path(policy["install_root"]) / "lib" / runtime_component["filename"] + runtime_path.chmod(0o644) + runtime_path.write_bytes(b"changed") + with isolated_test_install_trust(), mock.patch.object( + run_matrix, "run_approved_executable") as execute, self.assertRaisesRegex( + run_matrix.TraceError, "runtime component .* (immutable|SHA-256)"): run_matrix.prepare_prompt( builder=builder, builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, @@ -1532,9 +1879,11 @@ def test_prompt_builder_rejects_runtime_receipt_before_execution(self) -> None: def test_prompt_builder_rejects_output_binary_and_corpus_mutation(self) -> None: for mutation, message in ( ("output", "output differs from external approval"), - ("builder", "SHA-256 differs from external approval"), + ("builder", "immutable|SHA-256 differs from external approval"), ("corpus", "corpus changed during execution"), - ("runtime", "runtime component SHA-256 differs from external approval"), + ("runtime", "runtime component .*immutable|runtime component SHA-256 differs"), + ("runtime-load", "loaded runtime libraries differ from external approval"), + ("tokenizer", "tokenizer policy"), ): with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: root = Path(temp).resolve() @@ -1563,23 +1912,45 @@ def test_prompt_builder_rejects_output_binary_and_corpus_mutation(self) -> None: TEST_PROMPT_BUILDER_POLICY_ID, policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, ) - initial_identity = run_matrix.approved_executable_identity( - builder, - expected_path=policy["executable_path"], - expected_sha256=policy["executable_sha256"], - label="prompt builder", - ) + with isolated_test_install_trust(): + initial_identity = run_matrix.approved_executable_identity( + builder, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="prompt builder", + ) def run_builder(command, **_kwargs): + if "--dsv41-attest-build" in command: + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps(fixture_runtime_build(policy)), + "", + ), + initial_identity, + ) output.write_bytes(b"prompt") if mutation == "builder": + builder.chmod(0o755) builder.write_bytes(b"changed") if mutation == "corpus": corpus.write_bytes(b"changed") if mutation == "runtime": runtime_component = policy["runtime_receipt"]["components"][0] - (Path(policy["install_root"]) / "lib" / runtime_component["filename"]).write_bytes( - b"changed") + runtime_path = Path(policy["install_root"]) / "lib" / runtime_component["filename"] + runtime_path.chmod(0o644) + runtime_path.write_bytes(b"changed") + runtime_build = fixture_runtime_build(policy) + tokenizer = copy.deepcopy(policy["tokenizer"]) + if mutation == "runtime-load": + runtime_build["runtime_libraries"][0]["path"] = "/tmp/unapproved.so" + runtime_build["runtime_libraries_post"][0]["path"] = "/tmp/unapproved.so" + if mutation == "tokenizer": + tokenizer["parse_special"] = False return ( subprocess.CompletedProcess( command, @@ -1588,7 +1959,8 @@ def run_builder(command, **_kwargs): "target_tokens": 2, "actual_tokens": 2, "byte_count": 6, - "add_bos": True, + "tokenizer": tokenizer, + "runtime_build": runtime_build, "temporary_directory": str(tmpdir), }), "", @@ -1596,7 +1968,8 @@ def run_builder(command, **_kwargs): initial_identity, ) - with mock.patch.dict(os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( + with isolated_test_install_trust(), mock.patch.dict( + os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( run_matrix, "run_approved_executable", side_effect=run_builder), self.assertRaisesRegex( (RuntimeError, run_matrix.TraceError), message): run_matrix.prepare_prompt( @@ -1628,6 +2001,133 @@ def test_signed_approval_binding_tamper_is_rejected(self) -> None: trace.TraceError, "candidate exporter approval differs from external policy"): self._seal_test_bundle(root) + def test_tokenizer_policy_contradictions_are_rejected_before_signing(self) -> None: + mutations = { + "add-bos-and-parse-special": { + "add_bos": False, + "parse_special": False, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": False, + "require_round_trip": True, + }, + "add-bos": { + "add_bos": False, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": False, + "require_round_trip": True, + }, + "detokenize-special": { + "add_bos": True, + "parse_special": True, + "detokenize_special": False, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": True, + }, + "bos-removal": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": False, + "require_round_trip": True, + }, + "round-trip": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": False, + }, + } + for name, tokenizer in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + record = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + record["config"]["tokenizer"] = tokenizer + (root / trace.MANIFEST_NAME).write_text( + trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "tokenizer"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + valid = fixture_prompt_builder_policy(b"prompt")["tokenizer"] + malformed = { + "missing": {key: value for key, value in valid.items() if key != "parse_special"}, + "extra": {**valid, "unknown": False}, + "non-boolean": {**valid, "add_bos": 1}, + } + for name, tokenizer in malformed.items(): + with self.subTest(name=name), self.assertRaisesRegex(trace.TraceError, "tokenizer policy"): + trace.validate_tokenizer_policy(tokenizer) + + def test_prompt_provenance_tokenizer_mutation_is_rejected_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + old_path = root / record["prompt"]["provenance"]["path"] + provenance = trace.strict_json_loads(old_path.read_text(encoding="ascii")) + provenance["tokenizer"]["add_bos"] = False + provenance["tokenizer"]["remove_leading_bos_before_detokenize"] = False + data = (trace.canonical_json(provenance) + "\n").encode("ascii") + digest = trace.sha256_bytes(data) + new_path = root / "provenance" / f"{digest}.json" + new_path.write_bytes(data) + old_path.unlink() + record["prompt"]["provenance"] = { + "path": f"provenance/{digest}.json", + "sha256": digest, + } + manifest_path.write_text(trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "prompt provenance tokenizer"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + def test_prompt_provenance_runtime_closure_mutation_is_rejected_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + old_path = root / record["prompt"]["provenance"]["path"] + provenance = trace.strict_json_loads(old_path.read_text(encoding="ascii")) + provenance["builder_runtime_build"]["runtime_libraries"][0]["path"] = "/tmp/unapproved.so" + provenance["builder_runtime_build"]["runtime_libraries_post"][0]["path"] = "/tmp/unapproved.so" + provenance["builder_runtime_build_sha256"] = trace.sha256_bytes( + trace.canonical_json(provenance["builder_runtime_build"]).encode("ascii")) + data = (trace.canonical_json(provenance) + "\n").encode("ascii") + digest = trace.sha256_bytes(data) + new_path = root / "provenance" / f"{digest}.json" + new_path.write_bytes(data) + old_path.unlink() + record["prompt"]["provenance"] = { + "path": f"provenance/{digest}.json", + "sha256": digest, + } + manifest_path.write_text(trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "loaded runtime libraries"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + def test_authorization_tokenizer_hash_mutation_is_rejected_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + record["authorization"]["tokenizer_policy_sha256"] = "f" * 64 + manifest_path.write_text(trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "tokenizer policy differs"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + def test_seal_rejects_protected_bundle_mutations(self) -> None: with tempfile.TemporaryDirectory() as temp: baseline = Path(temp) / "baseline" @@ -3806,14 +4306,27 @@ def test_prompt_builder_result_becomes_strict_provenance(self) -> None: TEST_PROMPT_BUILDER_POLICY_ID, policies={TEST_PROMPT_BUILDER_POLICY_ID: builder_policy}, ) - builder_identity = run_matrix.approved_executable_identity( - builder, - expected_path=builder_policy["executable_path"], - expected_sha256=builder_policy["executable_sha256"], - label="prompt builder", - ) + with isolated_test_install_trust(): + builder_identity = run_matrix.approved_executable_identity( + builder, + install_root=builder_policy["install_root"], + expected_owner_uid=builder_policy["install_owner_uid"], + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + ) def run_builder(command, **_kwargs): + if "--dsv41-attest-build" in command: + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps(fixture_runtime_build(builder_policy)), + "", + ), + builder_identity, + ) output.write_bytes(b"prompt") return ( subprocess.CompletedProcess( @@ -3823,7 +4336,8 @@ def run_builder(command, **_kwargs): "target_tokens": 2, "actual_tokens": 2, "byte_count": 6, - "add_bos": True, + "tokenizer": builder_policy["tokenizer"], + "runtime_build": fixture_runtime_build(builder_policy), "temporary_directory": str(tmpdir.resolve()), }), "", @@ -3831,7 +4345,8 @@ def run_builder(command, **_kwargs): builder_identity, ) - with mock.patch.dict(os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( + with isolated_test_install_trust(), mock.patch.dict( + os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( run_matrix, "run_approved_executable", side_effect=run_builder), mock.patch.object( sys, "stderr", io.StringIO()): result = run_matrix.prepare_prompt( @@ -3858,6 +4373,9 @@ def run_builder(command, **_kwargs): "decode_steps", "builder_approval_id", "builder_approval_sha256", "builder_path", "builder_sha256", "builder_revision", "builder_runtime_profile", "target_tokens", "actual_tokens", + "tokenizer", "builder_runtime_build", "builder_runtime_build_sha256", + "builder_install_trust", + "builder_install_trust_sha256", }, ) preflight.validate_prompt_provenance( @@ -4539,6 +5057,7 @@ def test_local_base_regression_requires_attested_oracle_revision(self) -> None: "candidate_exporter", TEST_CANDIDATE_EXPORTER_POLICY_ID, candidate_policy_sha256, + base_manifest["candidate"]["install_trust_sha256"], ) ) with trace.TraceBundleWriter(base, base_manifest) as writer: diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index 983284d363ac..c47133812cc2 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -117,8 +117,11 @@ endif() set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) add_executable(${PROMPT_TARGET} prompt-builder.cpp) -target_link_libraries(${PROMPT_TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) +add_dependencies(${PROMPT_TARGET} dsv41-runtime-receipt) +target_link_libraries(${PROMPT_TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) +target_include_directories(${PROMPT_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_compile_features(${PROMPT_TARGET} PRIVATE cxx_std_17) +target_compile_definitions(${PROMPT_TARGET} PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") if(NOT WIN32) set_target_properties( ${PROMPT_TARGET} @@ -131,18 +134,22 @@ if(LLAMA_TOOLS_INSTALL) install( TARGETS ${TARGET} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - COMPONENT ${DSV41_INSTALL_COMPONENT}) + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) install( TARGETS ${DSV41_RECEIPT_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE NAMELINK_COMPONENT ${DSV41_INSTALL_COMPONENT}) install( TARGETS ${PROMPT_TARGET} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - COMPONENT ${DSV41_INSTALL_COMPONENT}) + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) endif() if(LLAMA_BUILD_TESTS) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 6e7075538b97..bd06c887ba06 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -15,9 +15,13 @@ Seal v1 signs `dsv41-trace-bundle-v1\n` followed by canonical JSON records for ` Signer trust is external to the bundle. `APPROVED_TRACE_SIGNERS` maps one restricted ASCII principal to one exact OpenSSH Ed25519 public key, runtime lane, and runtime profile, and is intentionally empty until a separately authorized run. Candidate and oracle signers are not interchangeable. Every signed manifest also binds an externally supplied 256-bit challenge, lane-specific run ID, and bounded validity window. Validation requires those expected values from outside the bundle and rejects missing, mismatched, reused within a comparison, cross-lane, not-yet-valid, or expired authorization. The bundle cannot provide a public key, authoritative principal, verifier path, or allowed-signers file. Validation uses only `/usr/bin/ssh-keygen` on macOS and Linux or `C:\Windows\System32\OpenSSH\ssh-keygen.exe` on Windows, rejects symlinks and unsupported `-Y` implementations, clears SSH-agent influence, and never searches `PATH`. The private signing key must be an owned restrictive regular file outside the bundle and is never copied or logged. Tests use explicit test-only verifiers with ephemeral lane-specific keys; production validation does not trust those keys. -Executable trust is also external to the bundle and to the reviewed source revision. `APPROVED_EXECUTABLE_APPROVERS`, `APPROVED_CANDIDATE_EXPORTERS`, and `APPROVED_PROMPT_BUILDERS` are intentionally empty in production source. An authorized run must receive a canonical detached approval policy plus its OpenSSH signature and an externally expected approver principal. Both files must be absolute canonical non-symlinked regular files outside every protected output root. The approver public key comes only from `APPROVED_EXECUTABLE_APPROVERS`; the fixed `ssh-keygen` verifies namespace `dsv41-executable-approval-v1` before either executable can run. This avoids an impossible same-revision self-reference: the policy records the artifact producer revision and exact executable hashes, while a separate `verifier_revision` records the harness revision that consumes the policy. +Executable trust is also external to the bundle and to the reviewed source revision. `APPROVED_EXECUTABLE_APPROVERS`, `APPROVED_CANDIDATE_EXPORTERS`, and `APPROVED_PROMPT_BUILDERS` are intentionally empty in production source. An authorized run must receive a canonical detached approval policy plus its OpenSSH signature and an externally expected approver principal. Both files must be absolute canonical non-symlinked one-link regular files outside every protected output root. Their trusted root and owner come only from `APPROVED_EXECUTABLE_APPROVERS`; the fixed `ssh-keygen` verifies namespace `dsv41-executable-approval-v2` before either executable can run. This avoids an impossible same-revision self-reference: the policy records the artifact producer revision and exact executable hashes, while a separate `verifier_revision` records the harness revision that consumes the policy. -The candidate exporter approval binds the exact producer revision, base revision, binary diff SHA-256, canonical install root and exporter path, exporter SHA-256, runtime profile, and complete embedded runtime receipt. The prompt-builder approval binds its producer revision, canonical install/source roots and executable path, executable SHA-256, exact runtime profile and receipt, model and corpus identities, and exact prompt hash, byte count, BOS behavior, context, and decode configuration for every authorized case. Every receipt library is hashed before the corresponding executable can run and rechecked afterward. On the Linux candidate host, both the exporter and prompt builder are opened without following the final symlink and executed through the retained `/proc/self/fd` descriptor, so pathname replacement cannot select a different executable object. The signed execution authorization records the complete approval-policy SHA-256, verifier revision, and both selected approval record IDs and hashes. Candidate-derived attestations and prompt provenance are evidence only and must exactly equal those external records. +The candidate exporter approval binds the exact producer revision, base revision, binary diff SHA-256, canonical install root and exporter path, exporter SHA-256, runtime profile, and complete embedded runtime receipt. The prompt-builder approval binds its producer revision, canonical install/source roots and executable path, executable SHA-256, exact runtime profile and receipt, model and corpus identities, complete tokenizer policy, and exact prompt hash, byte count, context, and decode configuration for every authorized case. The tokenizer policy explicitly binds `add_bos`, `parse_special`, `detokenize_special`, leading-BOS removal, and exact token round-trip. The builder invocation, native result, signed provenance, execution authorization, candidate environment, and candidate manifest must all agree with it. + +Every approved install tree is verified before launch. The install owner is externally selected and must differ from the unprivileged execution identity. Every ancestor is canonical, non-symlinked, trusted-owned, ACL-free, and not writable by the execution identity or group/other. Every approved executable and receipt library is a canonical, ACL-free, non-writable, one-link regular file with the exact owner and SHA-256. On Linux, both the exporter and prompt builder are opened without following the final symlink and executed through the retained `/proc/self/fd` descriptor. Receipt-library descriptors remain open through process completion, while the immutable canonical install hierarchy prevents pathname substitution during loader consumption. Both native tools report the actual canonical loaded project-library paths and hashes; the prompt provenance and candidate manifest require the loaded closure to equal the approved receipt before and after protected work. Ordinary user-owned install smoke verifies layout, RPATH, and bytes only; it does not establish production trusted-root authorization. + +The signed execution authorization records the complete approval-policy SHA-256, verifier revision, tokenizer-policy SHA-256, and both selected approval record IDs, policy hashes, and install-trust hashes. Candidate-derived attestations and prompt provenance are evidence only and must exactly equal those external records. Seal creation performs the same semantic manifest, provenance, event, and coverage validation as standalone verification before invoking the signer. Every manifest and memory audit declares that the expert cache and KV cache are memory-resident and that there are no external cache or state paths. Missing, substituted, or additional file-backed cache/state declarations fail closed. @@ -162,7 +166,7 @@ The launcher checks the approved candidate exporter path, hash, device/inode, si Production installs no manifest-writing or runtime-path probe option. With `LLAMA_BUILD_TESTS`, CMake builds the non-installed `test-deepseek41-trace-manifest` harness from the same native writer implementation. Its input accepts only model, prompt, audit, expected-coverage, and event-count data. Runtime, accelerator, path, configuration, build, and environment evidence comes from measured local state, and CPU or Darwin output is explicitly test-only and cannot satisfy production candidate or signer requirements. The trace install component places the exact receipt libraries beside the tools and sets `@loader_path/../lib` on macOS or `$ORIGIN/../lib` on ELF so installed `--version` needs no loader override. The install test also hashes every installed component and requires exact equality with the receipt embedded after the original library link; install-time rewriting or re-signing fails. -`run_matrix.py --llama-only` copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, and verifies the approved builder path/hash and both original and copied corpus identities before prompt construction. It rechecks the builder and corpus identities after execution and requires the generated prompt hash, byte count, BOS behavior, context, and decode configuration to equal the signed external approval before writing provenance. Pass the detached approval policy and signature, selected candidate and prompt approval IDs, both executables, the producer revision/base/diff identity, and the verifier checkout. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. +`run_matrix.py --llama-only` runs the approved prompt builder on the Linux candidate host, copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, and verifies the approved builder path/hash, loaded runtime closure, complete tokenizer policy, and both original and copied corpus identities before prompt construction. It rechecks the builder, loaded libraries, and corpus identities after execution and requires the generated prompt hash, byte count, tokenizer policy, context, and decode configuration to equal the signed external approval before writing provenance. Pass the detached approval policy and signature, selected candidate and prompt approval IDs, both executables, the producer revision/base/diff identity, and the verifier checkout. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. ## Apple Metal oracle execution gate diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index c16f2725199c..c939d2c5ade1 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -1505,6 +1505,25 @@ int main(int argc, char ** argv) { if (required_environment("HIP_LAUNCH_BLOCKING") != "1") { throw std::runtime_error("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs"); } + const json tokenizer_policy = json::parse(required_environment("DSV41_TOKENIZER_POLICY")); + if (!tokenizer_policy.is_object() || tokenizer_policy.size() != 5 || + !tokenizer_policy.contains("add_bos") || + !tokenizer_policy.contains("parse_special") || + !tokenizer_policy.contains("detokenize_special") || + !tokenizer_policy.contains("remove_leading_bos_before_detokenize") || + !tokenizer_policy.contains("require_round_trip") || + !tokenizer_policy["add_bos"].is_boolean() || + !tokenizer_policy["parse_special"].is_boolean() || + !tokenizer_policy["detokenize_special"].is_boolean() || + !tokenizer_policy["remove_leading_bos_before_detokenize"].is_boolean() || + !tokenizer_policy["require_round_trip"].is_boolean() || + !tokenizer_policy["parse_special"].get() || + !tokenizer_policy["detokenize_special"].get() || + !tokenizer_policy["require_round_trip"].get() || + tokenizer_policy["remove_leading_bos_before_detokenize"].get() != + tokenizer_policy["add_bos"].get()) { + throw std::runtime_error("DSV41_TOKENIZER_POLICY is not the exact approved tokenizer policy"); + } if (params.devices.size() != 1 || params.devices[0] == nullptr) { throw std::runtime_error("trace tool requires exactly one selected execution device"); } @@ -1561,7 +1580,11 @@ int main(int argc, char ** argv) { } const llama_vocab * vocab = llama_model_get_vocab(model); const bool add_bos = llama_vocab_get_add_bos(vocab); - const std::vector tokens = common_tokenize(ctx, params.prompt, add_bos, true); + if (add_bos != tokenizer_policy["add_bos"].get()) { + throw std::runtime_error("model tokenizer add_bos differs from the approved policy"); + } + const std::vector tokens = common_tokenize( + ctx, params.prompt, add_bos, tokenizer_policy["parse_special"].get()); const int32_t n_vocab = llama_vocab_n_tokens(vocab); if (tokens.empty()) { throw std::runtime_error("prompt tokenization produced no tokens"); @@ -1602,8 +1625,7 @@ int main(int argc, char ** argv) { {"load_mode", static_cast(params.load_mode)}, {"expert_cache_slots", params.expert_cache_slots}, {"expert_cache_bytes", static_cast(params.expert_cache_mib) << 20}, - {"tokenizer_add_bos", add_bos}, - {"tokenizer_parse_special", true}, + {"tokenizer", tokenizer_policy}, {"deepseek41", { {"layer_count", 40}, {"vocab_size", n_vocab}, diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index 3511063aa9ff..d939dd26f478 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -15,7 +15,15 @@ from pathlib import Path from typing import Callable -from trace_format import NO_EXTERNAL_STATE_STORAGE, TraceError, validate_watchdog_event +from trace_format import ( + NO_EXTERNAL_STATE_STORAGE, + TraceError, + install_trust_sha256, + runtime_build_evidence_sha256, + validate_install_trust_evidence, + validate_runtime_build_evidence, + validate_watchdog_event, +) FORBIDDEN_ROOT = Path("/mnt/bigspace") SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 @@ -1376,12 +1384,32 @@ def validate_prompt_provenance( "builder_sha256": builder_policy["executable_sha256"], "builder_revision": builder_policy["revision"], "builder_runtime_profile": builder_policy["runtime_profile"], + "tokenizer": builder_policy["tokenizer"], } for key, value in expected.items(): if record.get(key) != value: raise PreflightError(f"prompt provenance {key} mismatch") - if set(record) != set(expected): + required = set(expected) | { + "builder_runtime_build", + "builder_runtime_build_sha256", + "builder_install_trust", + "builder_install_trust_sha256", + } + if set(record) != required: raise PreflightError("prompt provenance fields are invalid") + try: + runtime_build = validate_runtime_build_evidence( + record["builder_runtime_build"], builder_policy, label="prompt builder") + runtime_build_sha256 = runtime_build_evidence_sha256( + runtime_build, builder_policy, label="prompt builder") + trust = validate_install_trust_evidence(record["builder_install_trust"], builder_policy) + trust_sha256 = install_trust_sha256(trust) + except TraceError as error: + raise PreflightError(f"prompt provenance runtime trust is invalid: {error}") from error + if record["builder_runtime_build_sha256"] != runtime_build_sha256: + raise PreflightError("prompt provenance runtime build SHA-256 mismatch") + if record["builder_install_trust_sha256"] != trust_sha256: + raise PreflightError("prompt provenance install trust SHA-256 mismatch") matches = [ prompt_record for prompt_record in builder_policy["prompts"] if prompt_record["corpus_name"] == corpus_name and diff --git a/tools/deepseek-v41-trace/prompt-builder.cpp b/tools/deepseek-v41-trace/prompt-builder.cpp index 47b0c700785e..4f1b7cb63b7e 100644 --- a/tools/deepseek-v41-trace/prompt-builder.cpp +++ b/tools/deepseek-v41-trace/prompt-builder.cpp @@ -1,21 +1,286 @@ +#include "build-info.h" #include "common.h" +#include "ggml-backend.h" +#include "ggml.h" #include "host-attestation.h" +#include "dsv41-runtime-receipt.h" +extern "C" { +#include "hash/sha256/sha256.h" +} #include "llama.h" #include +#include +#include +#include #include #include #include #include #include +#include +#include +#include +#include #include #include #include +#if defined(__linux__) +#include +#include +#include +#endif + namespace fs = std::filesystem; using json = nlohmann::ordered_json; +#if defined(__linux__) +static constexpr const char * BUILD_REVISION = DSV41_BUILD_REVISION; +#endif + +static std::string sha256_hex(const unsigned char digest[SHA256_DIGEST_SIZE]) { + std::ostringstream stream; + stream << std::hex << std::setfill('0'); + for (size_t i = 0; i < SHA256_DIGEST_SIZE; ++i) { + stream << std::setw(2) << static_cast(digest[i]); + } + return stream.str(); +} + +static std::string sha256_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open for SHA-256: " + path.string()); + } + sha256_t state; + sha256_init(&state); + std::vector buffer(1024 * 1024); + while (input) { + input.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + const std::streamsize count = input.gcount(); + if (count > 0) { + sha256_update(&state, buffer.data(), static_cast(count)); + } + } + if (!input.eof()) { + throw std::runtime_error("failed while hashing: " + path.string()); + } + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_final(&state, digest); + return sha256_hex(digest); +} + +static fs::path canonical_path(const fs::path & path, const char * label) { + try { + return fs::canonical(path); + } catch (const fs::filesystem_error & error) { + throw std::runtime_error(std::string("cannot resolve ") + label + ": " + error.what()); + } +} + +#if defined(__linux__) +static fs::path current_executable_path() { + return canonical_path("/proc/self/exe", "current executable"); +} + +template +static const void * function_address(T function) { + return reinterpret_cast(reinterpret_cast(function)); +} + +static fs::path module_path(const void * address) { + Dl_info info = {}; + if (dladdr(address, &info) == 0 || info.dli_fname == nullptr || info.dli_fname[0] == '\0') { + throw std::runtime_error("cannot identify loaded runtime module"); + } + return canonical_path(info.dli_fname, "loaded runtime module"); +} + +static bool is_project_runtime_library(const fs::path & path) { + std::string name = path.filename().string(); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { + return static_cast(std::tolower(value)); + }); + return name.rfind("libllama", 0) == 0 || name.rfind("libggml", 0) == 0 || + name.rfind("ggml", 0) == 0; +} + +static bool is_trusted_system_runtime_path(const fs::path & path) { + const std::string value = path.string(); + std::error_code error; + const fs::path rocm_root = fs::canonical("/opt/rocm", error); + if (!error) { + const std::string prefix = rocm_root.string() + "/"; + if (value.rfind(prefix, 0) == 0) { + for (fs::path current = path; current != rocm_root.parent_path(); current = current.parent_path()) { + struct stat status = {}; + if (::stat(current.c_str(), &status) != 0 || status.st_uid != 0 || + (status.st_mode & (S_IWGRP | S_IWOTH)) != 0) { + return false; + } + if (current == rocm_root) { + return true; + } + } + } + } + return value.rfind("/lib/", 0) == 0 || + value.rfind("/lib64/", 0) == 0 || + value.rfind("/usr/lib/", 0) == 0 || + value.rfind("/usr/lib64/", 0) == 0; +} + +static std::set loaded_runtime_images(const fs::path & executable) { + std::set result; + struct image_context { + const fs::path * executable; + std::set * result; + std::string error; + } context = {&executable, &result, {}}; + const auto callback = [](dl_phdr_info * info, size_t, void * data) { + image_context & context = *static_cast(data); + if (!context.error.empty() || info->dlpi_name == nullptr || info->dlpi_name[0] == '\0') { + return 0; + } + try { + const fs::path reported_path = info->dlpi_name; + if (!reported_path.is_absolute() && + (reported_path == "linux-vdso.so.1" || reported_path == "linux-gate.so.1")) { + return 0; + } + const fs::path path = canonical_path(reported_path, "loaded runtime module"); + if (path != *context.executable) { + context.result->insert(path); + } + } catch (const std::exception & error) { + context.error = error.what(); + } + return 0; + }; + if (dl_iterate_phdr(callback, &context) < 0 || !context.error.empty()) { + throw std::runtime_error( + context.error.empty() ? "cannot enumerate loaded runtime modules" : context.error); + } + return result; +} + +static fs::path runtime_library_directory(const fs::path & executable) { + if (std::string(dsv41_runtime_receipt::profile) != "sibling-lib") { + throw std::runtime_error("prompt builder requires the sibling-lib runtime profile"); + } + return executable.parent_path().parent_path() / "lib"; +} + +static void load_runtime_backends(const fs::path & executable) { +#if defined(GGML_BACKEND_DL) + const std::string directory = runtime_library_directory(executable).string(); + ggml_backend_load_all_from_path(directory.c_str()); +#else + (void) executable; + ggml_backend_load_all(); +#endif +} + +static json runtime_libraries_json(const fs::path & executable) { + if (dsv41_runtime_receipt::entries.size() != dsv41_runtime_receipt::components.size()) { + throw std::runtime_error("embedded runtime profile size differs from the receipt"); + } + const fs::path library_directory = + canonical_path(runtime_library_directory(executable), "runtime library directory"); + std::map receipt_by_component; + std::map receipt_by_path; + for (size_t index = 0; index < dsv41_runtime_receipt::entries.size(); ++index) { + const dsv41_runtime_receipt::entry & entry = dsv41_runtime_receipt::entries[index]; + if (std::string(entry.component) != dsv41_runtime_receipt::components[index]) { + throw std::runtime_error("embedded runtime profile differs from the receipt"); + } + const fs::path expected = + canonical_path(library_directory / entry.filename, "receipt runtime module"); + if (!receipt_by_component.emplace(entry.component, &entry).second || + !receipt_by_path.emplace(expected, &entry).second) { + throw std::runtime_error("embedded runtime receipt is not canonical and unique"); + } + } + std::set libraries; + for (const fs::path & image : loaded_runtime_images(executable)) { + if (image.parent_path() == library_directory || is_project_runtime_library(image)) { + libraries.insert(image); + } else if (!is_trusted_system_runtime_path(image)) { + throw std::runtime_error("loaded unclassified module outside trusted system roots: " + image.string()); + } + } + std::map loaded_by_component; + for (const fs::path & library : libraries) { + const auto receipt = receipt_by_path.find(library); + if (library.parent_path() != library_directory || receipt == receipt_by_path.end()) { + throw std::runtime_error("loaded project runtime module is absent from the receipt: " + library.string()); + } + if (!loaded_by_component.emplace(receipt->second->component, library).second || + sha256_file(library) != receipt->second->sha256) { + throw std::runtime_error("loaded runtime module identity differs from the receipt"); + } + } + if (loaded_by_component.size() != receipt_by_component.size()) { + throw std::runtime_error("loaded runtime component set differs from the receipt"); + } + const std::array, 3> fixed_roles = {{ + {"llama-common", module_path(function_address(&llama_commit))}, + {"llama", module_path(function_address(&llama_model_load_from_file))}, + {"ggml-base", module_path(function_address(&ggml_init))}, + }}; + for (const auto & item : fixed_roles) { + const auto loaded = loaded_by_component.find(item.first); + if (loaded == loaded_by_component.end() || loaded->second != item.second) { + throw std::runtime_error( + "runtime symbol provider does not match receipt component " + std::string(item.first)); + } + } + json result = json::array(); + for (const fs::path & library : libraries) { + const dsv41_runtime_receipt::entry & receipt = *receipt_by_path.at(library); + std::string role = "runtime:" + std::string(receipt.component); + if (library == fixed_roles[0].second) { + role = "build-info"; + } else if (library == fixed_roles[1].second) { + role = "llama"; + } else if (library == fixed_roles[2].second) { + role = "ggml"; + } + result.push_back({ + {"component", receipt.component}, + {"filename", receipt.filename}, + {"path", library.string()}, + {"sha256", receipt.sha256}, + {"role", std::move(role)}, + {"revision", receipt.revision[0] == '\0' ? json(nullptr) : json(receipt.revision)}, + }); + } + return result; +} + +static json runtime_build_json(const fs::path & executable) { + if (std::string(BUILD_REVISION) != llama_commit() || std::string(BUILD_REVISION) != ggml_commit()) { + throw std::runtime_error("loaded runtime library revision differs from the prompt builder revision"); + } + const json runtime_libraries = runtime_libraries_json(executable); + return { + {"revision", BUILD_REVISION}, + {"path", executable.string()}, + {"sha256", sha256_file(executable)}, + {"runtime_profile", { + {"name", dsv41_runtime_receipt::profile}, + {"components", dsv41_runtime_receipt::components}, + {"selected_backend_component", "ggml-hip"}, + }}, + {"runtime_receipt_sha256", dsv41_runtime_receipt::sha256}, + {"runtime_libraries", runtime_libraries}, + {"runtime_libraries_post", runtime_libraries}, + }; +} +#endif + static std::string read_file(const fs::path & path) { std::ifstream input(path, std::ios::binary); if (!input) { @@ -33,6 +298,17 @@ static std::string argument(int argc, char ** argv, const std::string & name) { throw std::runtime_error("missing argument: " + name); } +static bool boolean_argument(int argc, char ** argv, const std::string & name) { + const std::string value = argument(argc, argv, name); + if (value == "true") { + return true; + } + if (value == "false") { + return false; + } + throw std::runtime_error(name + " must be true or false"); +} + static std::string model_architecture(const llama_model * model) { char buffer[128] = {}; if (llama_model_meta_val_str(model, "general.architecture", buffer, sizeof(buffer)) < 0) { @@ -43,13 +319,46 @@ static std::string model_architecture(const llama_model * model) { int main(int argc, char ** argv) { try { +#if !defined(__linux__) + (void) argc; + (void) argv; + throw std::runtime_error("prompt builder runtime attestation requires Linux"); +#else + if (argc == 2 && std::string(argv[1]) == "--dsv41-attest-build") { + const fs::path executable_path = current_executable_path(); + const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked prompt builder"); + if (invoked_path != executable_path) { + throw std::runtime_error("invoked prompt builder path does not match the running executable"); + } + llama_backend_init(); + load_runtime_backends(executable_path); + std::printf("%s\n", runtime_build_json(executable_path).dump().c_str()); + return 0; + } fs::path model_path = argument(argc, argv, "--model"); fs::path corpus_path = argument(argc, argv, "--corpus"); fs::path output_path = argument(argc, argv, "--output"); const int64_t target_tokens = std::stoll(argument(argc, argv, "--tokens")); + const bool expected_add_bos = boolean_argument(argc, argv, "--tokenizer-add-bos"); + const bool parse_special = boolean_argument(argc, argv, "--tokenizer-parse-special"); + const bool detokenize_special = boolean_argument(argc, argv, "--tokenizer-detokenize-special"); + const bool remove_leading_bos = boolean_argument(argc, argv, "--tokenizer-remove-leading-bos"); + const bool require_round_trip = boolean_argument(argc, argv, "--tokenizer-require-round-trip"); if (target_tokens < 2) { throw std::runtime_error("--tokens must be at least 2"); } + if (!parse_special || !detokenize_special || !require_round_trip || + remove_leading_bos != expected_add_bos) { + throw std::runtime_error("prompt builder requires the exact approved tokenizer policy"); + } + const fs::path executable_path = current_executable_path(); + const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked prompt builder"); + if (invoked_path != executable_path) { + throw std::runtime_error("invoked prompt builder path does not match the running executable"); + } + llama_backend_init(); + load_runtime_backends(executable_path); + const json runtime_build = runtime_build_json(executable_path); model_path = dsv41::require_nvme_path(model_path, "model").resolved_path; corpus_path = dsv41::require_nvme_path(corpus_path, "corpus").resolved_path; output_path = dsv41::require_nvme_path(output_path, "prompt output").resolved_path; @@ -69,7 +378,6 @@ int main(int argc, char ** argv) { throw std::runtime_error("corpus is empty"); } - llama_backend_init(); llama_model_params model_params = llama_model_default_params(); model_params.vocab_only = true; llama_model * model = llama_model_load_from_file(model_path.string().c_str(), model_params); @@ -82,29 +390,33 @@ int main(int argc, char ** argv) { } const llama_vocab * vocab = llama_model_get_vocab(model); const bool add_bos = llama_vocab_get_add_bos(vocab); + if (add_bos != expected_add_bos) { + llama_model_free(model); + throw std::runtime_error("model tokenizer add_bos differs from the approved policy"); + } std::string repeated = corpus; - std::vector tokens = common_tokenize(vocab, repeated, add_bos, true); + std::vector tokens = common_tokenize(vocab, repeated, add_bos, parse_special); while (tokens.size() < static_cast(target_tokens)) { if (repeated.size() > (size_t(1) << 31)) { llama_model_free(model); throw std::runtime_error("repeated prompt exceeds 2 GiB"); } repeated += repeated; - tokens = common_tokenize(vocab, repeated, add_bos, true); + tokens = common_tokenize(vocab, repeated, add_bos, parse_special); } tokens.resize(static_cast(target_tokens)); std::vector content_tokens = tokens; - if (add_bos) { + if (remove_leading_bos) { if (content_tokens.front() != llama_vocab_bos(vocab)) { llama_model_free(model); throw std::runtime_error("tokenized prompt does not begin with the configured BOS token"); } content_tokens.erase(content_tokens.begin()); } - const std::string prompt = common_detokenize(vocab, content_tokens, true); - const std::vector verified = common_tokenize(vocab, prompt, add_bos, true); - if (verified != tokens) { + const std::string prompt = common_detokenize(vocab, content_tokens, detokenize_special); + const std::vector verified = common_tokenize(vocab, prompt, add_bos, parse_special); + if (require_round_trip && verified != tokens) { llama_model_free(model); throw std::runtime_error("constructed prompt does not round-trip to the target token IDs"); } @@ -118,15 +430,28 @@ int main(int argc, char ** argv) { throw std::runtime_error("cannot write prompt output"); } output.close(); + const json runtime_build_post = runtime_build_json(executable_path); + if (runtime_build_post != runtime_build) { + llama_model_free(model); + throw std::runtime_error("loaded runtime component set changed during prompt construction"); + } std::printf("%s\n", json({ {"target_tokens", target_tokens}, {"actual_tokens", verified.size()}, {"byte_count", prompt.size()}, - {"add_bos", add_bos}, + {"tokenizer", { + {"add_bos", expected_add_bos}, + {"parse_special", parse_special}, + {"detokenize_special", detokenize_special}, + {"remove_leading_bos_before_detokenize", remove_leading_bos}, + {"require_round_trip", require_round_trip}, + }}, + {"runtime_build", runtime_build_post}, {"temporary_directory", temporary_storage.resolved_path.string()}, }).dump().c_str()); llama_model_free(model); return 0; +#endif } catch (const std::exception & error) { std::fprintf(stderr, "error: %s\n", error.what()); return 1; diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 05670e03929b..dab92f874d41 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -44,6 +44,7 @@ sha256_bytes, sha256_file, strict_json_loads, + tokenizer_policy_sha256, validate_signing_identity, ) @@ -313,6 +314,33 @@ def main() -> int: args.prompt_builder_policy_id, policies=approval_policy.prompt_builders, ) + harness_repo = resolved(args.repo) + harness_revision = subprocess.check_output( + ["git", "-C", str(harness_repo), "rev-parse", "HEAD"], + stderr=subprocess.STDOUT, + ).decode("ascii").strip() + if harness_revision != approval_policy.verifier_revision: + raise PreflightError("ds4 verifier checkout differs from the external approval policy") + if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: + raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") + model_sha256 = sha256_file(resolved(args.model)) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + provenance = validate_prompt_provenance( + args.prompt_provenance, + prompt=args.prompt, + corpus_name=args.corpus_name, + corpus_sha256=args.corpus_sha256, + model_sha256=model_sha256, + target_tokens=args.context - args.decode_steps, + context=args.context, + decode_steps=args.decode_steps, + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, + path_resolver=lambda path, label: Path( + str(darwin_storage_attestation(path, label)["resolved_path"])), + ) authorization = execution_authorization( lane=ORACLE_LANE, challenge=args.execution_challenge, @@ -321,23 +349,16 @@ def main() -> int: expires_unix=args.authorization_expires_unix, approval_policy_sha256=approval_policy.sha256, verifier_revision=approval_policy.verifier_revision, + tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), approvals={ "prompt_builder": approval_binding( "prompt_builder", args.prompt_builder_policy_id, prompt_policy_sha256, + provenance["record"]["builder_install_trust_sha256"], ), }, ) - harness_repo = resolved(args.repo) - harness_revision = subprocess.check_output( - ["git", "-C", str(harness_repo), "rev-parse", "HEAD"], - stderr=subprocess.STDOUT, - ).decode("ascii").strip() - if harness_revision != approval_policy.verifier_revision: - raise PreflightError("ds4 verifier checkout differs from the external approval policy") - if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: - raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") exporter = resolved(args.exporter) if not exporter.is_file() or not os.access(exporter, os.X_OK): raise PreflightError(f"trace exporter is not executable: {exporter}") @@ -374,24 +395,6 @@ def main() -> int: print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) return 0 - model_sha256 = sha256_file(resolved(args.model)) - if model_sha256 != MODEL_SHA256: - raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") - provenance = validate_prompt_provenance( - args.prompt_provenance, - prompt=args.prompt, - corpus_name=args.corpus_name, - corpus_sha256=args.corpus_sha256, - model_sha256=model_sha256, - target_tokens=args.context - args.decode_steps, - context=args.context, - decode_steps=args.decode_steps, - builder_approval_id=args.prompt_builder_policy_id, - builder_policy=prompt_policy, - builder_policy_sha256=prompt_policy_sha256, - path_resolver=lambda path, label: Path( - str(darwin_storage_attestation(path, label)["resolved_path"])), - ) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") preflight_audit = preflight(args, accelerator=accelerator, runner=runner) diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index b4f4eab2cff7..2736715d318b 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -46,6 +46,8 @@ candidate_exporter_approval, canonical_json, execution_authorization, + install_trust_evidence, + install_trust_sha256, load_executable_approval_policy, reject_loader_overrides, run_approved_executable, @@ -53,8 +55,10 @@ sha256_bytes, sha256_file, strict_json_loads, + tokenizer_policy_sha256, prompt_builder_approval, validate_signing_identity, + validate_tokenizer_policy, verify_approved_executable_identity, verify_approved_runtime_file_identities, ) @@ -74,7 +78,8 @@ def candidate_attestation( approval_id: str, approval_sha256: str, approval: dict[str, object], - verifier_revision: str) -> dict[str, str]: + verifier_revision: str, + install_trust: dict[str, object]) -> dict[str, object]: repo = resolved(args.repo) exporter = resolved(exporter) observed_verifier_revision = git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() @@ -133,6 +138,8 @@ def candidate_attestation( **expected, "exporter_approval_id": approval_id, "exporter_approval_sha256": approval_sha256, + "install_trust": install_trust, + "install_trust_sha256": install_trust_sha256(install_trust), } @@ -272,6 +279,7 @@ def query_runtime_build_attestation( result, _identity = run_approved_executable( [str(exporter), "--dsv41-attest-build", device], path=exporter, + runtime_policy=approval, expected_path=approval["executable_path"], expected_sha256=approval["executable_sha256"], label="candidate exporter", @@ -402,6 +410,7 @@ def query_accelerator_attestation( result, _identity = run_approved_executable( [str(exporter), "--dsv41-attest-device", device], path=exporter, + runtime_policy=approval, expected_path=approval["executable_path"], expected_sha256=approval["executable_sha256"], label="candidate exporter", @@ -517,27 +526,6 @@ def main() -> int: args.prompt_builder_policy_id, policies=approval_policy.prompt_builders, ) - authorization = execution_authorization( - lane=CANDIDATE_LANE, - challenge=args.execution_challenge, - run_id=args.run_id, - issued_unix=args.authorization_issued_unix, - expires_unix=args.authorization_expires_unix, - approval_policy_sha256=approval_policy.sha256, - verifier_revision=approval_policy.verifier_revision, - approvals={ - "candidate_exporter": approval_binding( - "candidate_exporter", - args.candidate_exporter_policy_id, - candidate_policy_sha256, - ), - "prompt_builder": approval_binding( - "prompt_builder", - args.prompt_builder_policy_id, - prompt_policy_sha256, - ), - }, - ) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") validate_signing_identity( @@ -549,12 +537,15 @@ def main() -> int: exporter = args.exporter exporter_identity = approved_executable_identity( exporter, + install_root=candidate_policy["install_root"], + expected_owner_uid=candidate_policy["install_owner_uid"], expected_path=candidate_policy["executable_path"], expected_sha256=candidate_policy["executable_sha256"], label="candidate exporter", ) runtime_identities = approved_runtime_file_identities( candidate_policy, label="candidate exporter") + candidate_trust = install_trust_evidence(exporter_identity, runtime_identities) exporter_sha256 = exporter_identity.sha256 if args.candidate_revision != candidate_policy["revision"] or ( args.base_revision != candidate_policy["base_revision"]) or ( @@ -566,6 +557,47 @@ def main() -> int: raise PreflightError("candidate verifier checkout differs from the external approval policy") if str(repo) != prompt_policy["source_root"] or candidate_policy["revision"] != prompt_policy["revision"]: raise PreflightError("candidate repository or revision differs from prompt builder approval") + model_sha256 = sha256_file(resolved(args.model)) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + provenance = validate_prompt_provenance( + args.prompt_provenance, + prompt=args.prompt, + corpus_name=args.corpus_name, + corpus_sha256=args.corpus_sha256, + model_sha256=model_sha256, + target_tokens=args.context - args.decode_steps, + context=args.context, + decode_steps=args.decode_steps, + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, + ) + prompt_trust_sha256 = provenance["record"]["builder_install_trust_sha256"] + authorization = execution_authorization( + lane=CANDIDATE_LANE, + challenge=args.execution_challenge, + run_id=args.run_id, + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), + approvals={ + "candidate_exporter": approval_binding( + "candidate_exporter", + args.candidate_exporter_policy_id, + candidate_policy_sha256, + install_trust_sha256(candidate_trust), + ), + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + prompt_trust_sha256, + ), + }, + ) verify_approved_runtime_file_identities( runtime_identities, label="candidate exporter") pre_runtime_build = query_runtime_build_attestation( @@ -594,22 +626,6 @@ def main() -> int: print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) return 0 - model_sha256 = sha256_file(resolved(args.model)) - if model_sha256 != MODEL_SHA256: - raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") - provenance = validate_prompt_provenance( - args.prompt_provenance, - prompt=args.prompt, - corpus_name=args.corpus_name, - corpus_sha256=args.corpus_sha256, - model_sha256=model_sha256, - target_tokens=args.context - args.decode_steps, - context=args.context, - decode_steps=args.decode_steps, - builder_approval_id=args.prompt_builder_policy_id, - builder_policy=prompt_policy, - builder_policy_sha256=prompt_policy_sha256, - ) attestation = candidate_attestation( args, exporter, @@ -618,6 +634,7 @@ def main() -> int: candidate_policy_sha256, candidate_policy, approval_policy.verifier_revision, + candidate_trust, ) if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") @@ -648,6 +665,7 @@ def main() -> int: environment["DSV41_TRACE_MEMORY_AUDIT"] = pre_audits["memory"] environment["DSV41_TRACE_SWAP_AUDIT"] = pre_audits["swap"] environment["DSV41_TRACE_WATCHDOG_AUDIT"] = pre_audits["watchdog"] + environment["DSV41_TOKENIZER_POLICY"] = canonical_json(prompt_policy["tokenizer"]) command = build_command(args, exporter, output) print("exec:", shlex.join(command), file=sys.stderr) verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") @@ -656,6 +674,7 @@ def main() -> int: result, executed_identity = run_approved_executable( command, path=exporter, + runtime_policy=candidate_policy, expected_path=candidate_policy["executable_path"], expected_sha256=candidate_policy["executable_sha256"], label="candidate exporter", @@ -705,6 +724,11 @@ def main() -> int: bind_candidate_attestation( output, attestation, accelerator, exporter, exporter_sha256, candidate_policy) bind_execution_authorization(output, authorization) + unsealed_manifest = strict_json_loads( + safe_trace_path(output, "manifest.json").read_text(encoding="ascii")) + if not isinstance(unsealed_manifest, dict) or validate_tokenizer_policy( + unsealed_manifest.get("config", {}).get("tokenizer")) != prompt_policy["tokenizer"]: + raise PreflightError("candidate manifest tokenizer policy differs from external approval") seal_bundle( output, private_key=args.signing_key, diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index 67dd0c05a6a9..ac590179f811 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -27,13 +27,19 @@ approved_runtime_file_identities, candidate_exporter_approval, execution_authorization, + install_trust_evidence, + install_trust_sha256, load_executable_approval_policy, prompt_builder_approval, reject_loader_overrides, + runtime_build_evidence_sha256, run_approved_executable, sha256_file, strict_json_loads, + tokenizer_policy_sha256, validate_signing_identity, + validate_runtime_build_evidence, + validate_tokenizer_policy, verify_approved_executable_identity, verify_approved_runtime_file_identities, ) @@ -46,6 +52,29 @@ ) +def query_prompt_builder_runtime_build( + builder: Path, + builder_policy: dict[str, object]) -> dict[str, object]: + result, _identity = run_approved_executable( + [str(builder), "--dsv41-attest-build"], + path=builder, + runtime_policy=builder_policy, + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"prompt builder build attestation failed: {result.stderr.strip()}") + try: + record = strict_json_loads(result.stdout) + return validate_runtime_build_evidence(record, builder_policy, label="prompt builder") + except TraceError as error: + raise RuntimeError(f"prompt builder build attestation is invalid: {error}") from error + + def run(command: list[str]) -> None: displayed = list(command) if "--signing-key" in displayed: @@ -93,6 +122,8 @@ def prepare_prompt( ) builder_identity = approved_executable_identity( builder, + install_root=builder_policy["install_root"], + expected_owner_uid=builder_policy["install_owner_uid"], expected_path=builder_policy["executable_path"], expected_sha256=builder_policy["executable_sha256"], label="prompt builder", @@ -106,17 +137,26 @@ def prepare_prompt( source_identity = file_identity(source_corpus) corpus_identity = file_identity(corpus) verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") + tokenizer = validate_tokenizer_policy(builder_policy["tokenizer"]) + pre_runtime_build = query_prompt_builder_runtime_build(builder, builder_policy) command = [ str(builder), "--model", str(model), "--corpus", str(corpus), "--output", str(output), "--tokens", str(target_tokens), + "--tokenizer-add-bos", str(tokenizer["add_bos"]).lower(), + "--tokenizer-parse-special", str(tokenizer["parse_special"]).lower(), + "--tokenizer-detokenize-special", str(tokenizer["detokenize_special"]).lower(), + "--tokenizer-remove-leading-bos", str( + tokenizer["remove_leading_bos_before_detokenize"]).lower(), + "--tokenizer-require-round-trip", str(tokenizer["require_round_trip"]).lower(), ] print("exec:", " ".join(command), file=sys.stderr) result, executed_identity = run_approved_executable( command, path=builder, + runtime_policy=builder_policy, expected_path=builder_policy["executable_path"], expected_sha256=builder_policy["executable_sha256"], label="prompt builder", @@ -138,14 +178,19 @@ def prepare_prompt( except TraceError as error: raise RuntimeError(f"prompt builder returned invalid JSON: {error}") from error if not isinstance(native_record, dict) or set(native_record) != { - "target_tokens", "actual_tokens", "byte_count", "add_bos", "temporary_directory"}: + "target_tokens", "actual_tokens", "byte_count", "tokenizer", + "runtime_build", "temporary_directory"}: raise RuntimeError("prompt builder returned an invalid result schema") if native_record.get("target_tokens") != target_tokens or native_record.get("actual_tokens") != target_tokens: raise RuntimeError("prompt builder did not produce the requested token count") if type(native_record.get("byte_count")) is not int or native_record["byte_count"] != output.stat().st_size: raise RuntimeError("prompt builder byte count does not match its output") - if type(native_record.get("add_bos")) is not bool: - raise RuntimeError("prompt builder add_bos result is invalid") + if validate_tokenizer_policy(native_record.get("tokenizer")) != tokenizer: + raise RuntimeError("prompt builder tokenizer policy differs from external approval") + runtime_build = validate_runtime_build_evidence( + native_record.get("runtime_build"), builder_policy, label="prompt builder") + if runtime_build != pre_runtime_build: + raise RuntimeError("prompt builder runtime build changed during prompt construction") temporary_directory = native_record["temporary_directory"] expected_temporary_directory = os.environ.get("TMPDIR") if not expected_temporary_directory or temporary_directory != str(resolved(Path(expected_temporary_directory))): @@ -153,9 +198,9 @@ def prepare_prompt( prompt_sha256 = sha256_file(output) prompt_byte_count = output.stat().st_size if prompt_sha256 != expected_prompt["prompt_sha256"] or ( - prompt_byte_count != expected_prompt["prompt_byte_count"]) or ( - native_record["add_bos"] != expected_prompt["add_bos"]): + prompt_byte_count != expected_prompt["prompt_byte_count"]): raise RuntimeError("prompt builder output differs from external approval") + trust_evidence = install_trust_evidence(builder_identity, runtime_identities) record = { "format": "dsv41-prompt-provenance", "version": 1, @@ -173,6 +218,12 @@ def prepare_prompt( "builder_sha256": builder_identity.sha256, "builder_revision": builder_policy["revision"], "builder_runtime_profile": builder_policy["runtime_profile"], + "tokenizer": tokenizer, + "builder_runtime_build": runtime_build, + "builder_runtime_build_sha256": runtime_build_evidence_sha256( + runtime_build, builder_policy, label="prompt builder"), + "builder_install_trust": trust_evidence, + "builder_install_trust_sha256": install_trust_sha256(trust_evidence), "target_tokens": target_tokens, "actual_tokens": target_tokens, } @@ -252,12 +303,30 @@ def main() -> int: args.prompt_builder_policy_id, policies=approval_policy.prompt_builders, ) - approved_executable_identity( + candidate_identity = approved_executable_identity( args.llama_exporter, + install_root=candidate_policy["install_root"], + expected_owner_uid=candidate_policy["install_owner_uid"], expected_path=candidate_policy["executable_path"], expected_sha256=candidate_policy["executable_sha256"], label="candidate exporter", ) + candidate_runtime_identities = approved_runtime_file_identities( + candidate_policy, label="candidate exporter") + candidate_trust = install_trust_evidence( + candidate_identity, candidate_runtime_identities) + prompt_builder = args.llama_prompt_builder + prompt_identity = approved_executable_identity( + prompt_builder, + install_root=prompt_policy["install_root"], + expected_owner_uid=prompt_policy["install_owner_uid"], + expected_path=prompt_policy["executable_path"], + expected_sha256=prompt_policy["executable_sha256"], + label="prompt builder", + ) + prompt_runtime_identities = approved_runtime_file_identities( + prompt_policy, label="prompt builder") + prompt_trust = install_trust_evidence(prompt_identity, prompt_runtime_identities) if args.candidate_revision != candidate_policy["revision"] or ( args.base_revision != candidate_policy["base_revision"]) or ( args.candidate_diff_sha256 != candidate_policy["diff_sha256"]): @@ -270,16 +339,19 @@ def main() -> int: expires_unix=args.authorization_expires_unix, approval_policy_sha256=approval_policy.sha256, verifier_revision=approval_policy.verifier_revision, + tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), approvals={ "candidate_exporter": approval_binding( "candidate_exporter", args.candidate_exporter_policy_id, candidate_policy_sha256, + install_trust_sha256(candidate_trust), ), "prompt_builder": approval_binding( "prompt_builder", args.prompt_builder_policy_id, prompt_policy_sha256, + install_trust_sha256(prompt_trust), ), }, ) @@ -318,13 +390,6 @@ def main() -> int: repo=repo, busy_patterns=args.busy_pattern, ) - prompt_builder = args.llama_prompt_builder - approved_executable_identity( - prompt_builder, - expected_path=prompt_policy["executable_path"], - expected_sha256=prompt_policy["executable_sha256"], - label="prompt builder", - ) if str(repo) != prompt_policy["source_root"] or args.candidate_revision != prompt_policy["revision"]: raise PreflightError("matrix repository or revision differs from prompt builder approval") if output.exists() and any(output.iterdir()): diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 67ba4a3c9435..0933b0efdc4f 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -21,12 +21,12 @@ DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" APPROVED_EXPORTERS: dict[str, str] = {} APPROVED_TRACE_SIGNERS: dict[str, dict[str, str]] = {} -APPROVED_EXECUTABLE_APPROVERS: dict[str, str] = {} +APPROVED_EXECUTABLE_APPROVERS: dict[str, dict[str, Any]] = {} APPROVED_CANDIDATE_EXPORTERS: dict[str, dict[str, Any]] = {} APPROVED_PROMPT_BUILDERS: dict[str, dict[str, Any]] = {} EXECUTABLE_APPROVAL_FORMAT = "dsv41-executable-approval" -EXECUTABLE_APPROVAL_VERSION = 1 -EXECUTABLE_APPROVAL_NAMESPACE = "dsv41-executable-approval-v1" +EXECUTABLE_APPROVAL_VERSION = 2 +EXECUTABLE_APPROVAL_NAMESPACE = "dsv41-executable-approval-v2" SEAL_FORMAT = "dsv41-trace-bundle-signature" SEAL_VERSION = 1 SEAL_NAMESPACE = "dsv41-trace-bundle-v1" @@ -246,12 +246,17 @@ class BundleFileReceipt: @dataclass(frozen=True) class ExecutableFileReceipt: path: str + install_root: str device: int inode: int + owner_uid: int + mode: int + link_count: int byte_count: int modified_ns: int changed_ns: int sha256: str + path_chain: tuple[tuple[str, int, int, int, int], ...] @dataclass(frozen=True) @@ -310,21 +315,138 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def _execution_uid() -> int: + if os.name != "posix" or not hasattr(os, "geteuid"): + raise TraceError("immutable install paths require a POSIX execution identity") + return os.geteuid() + + +def _path_is_writable_by_execution_identity(path: Path) -> bool: + if _execution_uid() == 0: + raise TraceError("immutable install paths require an unprivileged POSIX execution identity") + try: + return os.access(path, os.W_OK, effective_ids=True) + except (OSError, TypeError, NotImplementedError) as error: + raise TraceError(f"cannot verify effective write access for {path}: {error}") from error + + +def _require_distinct_trusted_owner(expected_owner_uid: int) -> None: + execution_uid = _execution_uid() + if execution_uid == 0 or expected_owner_uid == execution_uid: + raise TraceError("approved install tree owner must be distinct from the unprivileged execution identity") + + +def _has_access_control_entries(path: Path) -> bool: + if sys.platform.startswith("linux"): + try: + attributes = os.listxattr(path, follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect access controls for {path}: {error}") from error + return any(name in {"system.posix_acl_access", "system.posix_acl_default"} for name in attributes) + if sys.platform == "darwin": + tool = Path("/bin/ls") + try: + tool_stat = tool.stat(follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect the fixed macOS ACL verifier: {error}") from error + if not stat.S_ISREG(tool_stat.st_mode) or tool_stat.st_uid != 0 or ( + stat.S_IMODE(tool_stat.st_mode) & 0o022): + raise TraceError("the fixed macOS ACL verifier is not trusted") + try: + result = subprocess.run( + [str(tool), "-lde", str(path)], + stdin=subprocess.DEVNULL, + check=False, + capture_output=True, + text=True, + timeout=30, + env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot inspect access controls for {path}: {error}") from error + if result.returncode != 0: + raise TraceError(f"cannot inspect access controls for {path}") + first_line = result.stdout.splitlines()[0] if result.stdout else "" + fields = first_line.split() + if not fields or len(fields[0]) < 10: + raise TraceError(f"access control output for {path} is invalid") + return "+" in fields[0] + raise TraceError("immutable install path access-control verification is unsupported on this platform") + + +def _immutable_path_chain( + path: Path, + *, + install_root: Path, + expected_owner_uid: int, + label: str, +) -> tuple[tuple[str, int, int, int, int], ...]: + if not path.is_absolute() or not install_root.is_absolute() or ( + path != install_root and install_root not in path.parents): + raise TraceError(f"{label} path is outside its approved install root") + try: + if str(path.resolve(strict=True)) != str(path) or str(install_root.resolve(strict=True)) != str(install_root): + raise TraceError(f"{label} path must be canonical and must not use aliases") + except OSError as error: + raise TraceError(f"cannot resolve {label} path: {error}") from error + if type(expected_owner_uid) is not int or expected_owner_uid < 0: + raise TraceError(f"{label} install owner is invalid") + _require_distinct_trusted_owner(expected_owner_uid) + result = [] + current = Path(path.anchor) + candidates = [current] + for part in path.parent.parts[1:]: + current /= part + candidates.append(current) + install_seen = Path(path.anchor) == install_root + for current in candidates: + try: + current_stat = current.stat(follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect {label} path: {error}") from error + if stat.S_ISLNK(current_stat.st_mode): + raise TraceError(f"{label} path must not use symlinks") + if not stat.S_ISDIR(current_stat.st_mode): + raise TraceError(f"{label} parent path is not a directory") + if current == install_root: + install_seen = True + if current_stat.st_uid not in {0, expected_owner_uid}: + raise TraceError(f"{label} path is not trusted-owned") + if install_seen and current_stat.st_uid != expected_owner_uid: + raise TraceError(f"{label} install tree owner differs from external approval") + if stat.S_IMODE(current_stat.st_mode) & 0o022 or ( + _path_is_writable_by_execution_identity(current)) or _has_access_control_entries(current): + raise TraceError(f"{label} path is mutable by the execution identity or an untrusted group") + result.append(( + str(current), + current_stat.st_dev, + current_stat.st_ino, + current_stat.st_uid, + stat.S_IMODE(current_stat.st_mode), + )) + if not install_seen: + raise TraceError(f"{label} path does not traverse its approved install root") + return tuple(result) + + def _approved_file_identity( path: Path, *, + install_root: Path, + expected_owner_uid: int, expected_path: str, expected_sha256: str, label: str, executable: bool, ) -> tuple[ExecutableFileReceipt, int]: - if not path.is_absolute() or str(path) != expected_path: + if not path.is_absolute() or str(path) != expected_path or not install_root.is_absolute(): raise TraceError(f"{label} path differs from external approval") - current = Path(path.anchor) - for part in path.parts[1:]: - current /= part - if current.is_symlink(): - raise TraceError(f"{label} path must not use symlinks") + path_chain = _immutable_path_chain( + path, + install_root=install_root, + expected_owner_uid=expected_owner_uid, + label=label, + ) flags = os.O_RDONLY flags |= getattr(os, "O_CLOEXEC", 0) flags |= getattr(os, "O_NOFOLLOW", 0) @@ -333,9 +455,12 @@ def _approved_file_identity( before = os.fstat(descriptor) except OSError as error: raise TraceError(f"cannot open {label}: {error}") from error - if not stat.S_ISREG(before.st_mode) or (executable and not os.access(path, os.X_OK)): + if not stat.S_ISREG(before.st_mode) or before.st_uid != expected_owner_uid or before.st_nlink != 1 or ( + stat.S_IMODE(before.st_mode) & 0o222) or ( + _path_is_writable_by_execution_identity(path)) or _has_access_control_entries(path) or ( + executable and not (stat.S_IMODE(before.st_mode) & 0o111)): os.close(descriptor) - raise TraceError(f"{label} is not an approved regular file") + raise TraceError(f"{label} is not an immutable trusted-owned one-link regular file") try: digest = hashlib.sha256() with os.fdopen(os.dup(descriptor), "rb") as stream: @@ -360,24 +485,33 @@ def _approved_file_identity( raise TraceError(f"{label} SHA-256 differs from external approval") return ExecutableFileReceipt( path=str(path), + install_root=str(install_root), device=after.st_dev, inode=after.st_ino, + owner_uid=after.st_uid, + mode=stat.S_IMODE(after.st_mode), + link_count=after.st_nlink, byte_count=after.st_size, modified_ns=after.st_mtime_ns, changed_ns=after.st_ctime_ns, sha256=digest_value, + path_chain=path_chain, ), descriptor def approved_executable_identity( path: Path, *, + install_root: str, + expected_owner_uid: int, expected_path: str, expected_sha256: str, label: str, ) -> ExecutableFileReceipt: identity, descriptor = _approved_file_identity( path, + install_root=Path(install_root), + expected_owner_uid=expected_owner_uid, expected_path=expected_path, expected_sha256=expected_sha256, label=label, @@ -395,6 +529,8 @@ def verify_approved_executable_identity( ) -> None: observed = approved_executable_identity( path, + install_root=expected.install_root, + expected_owner_uid=expected.owner_uid, expected_path=expected.path, expected_sha256=expected.sha256, label=label, @@ -414,6 +550,8 @@ def approved_runtime_file_identities( path = install_root / "lib" / component["filename"] identity, descriptor = _approved_file_identity( path, + install_root=install_root, + expected_owner_uid=policy["install_owner_uid"], expected_path=str(path), expected_sha256=component["sha256"], label=f"{label} runtime component {component['component']}", @@ -432,6 +570,8 @@ def verify_approved_runtime_file_identities( for identity in identities: observed, descriptor = _approved_file_identity( Path(identity.path), + install_root=Path(identity.path).parent.parent, + expected_owner_uid=identity.owner_uid, expected_path=identity.path, expected_sha256=identity.sha256, label=f"{label} runtime component", @@ -446,6 +586,7 @@ def run_approved_executable( command: list[str], *, path: Path, + runtime_policy: dict[str, Any], expected_path: str, expected_sha256: str, label: str, @@ -455,16 +596,38 @@ def run_approved_executable( raise TraceError(f"{label} descriptor execution requires Linux") identity, descriptor = _approved_file_identity( path, + install_root=Path(runtime_policy["install_root"]), + expected_owner_uid=runtime_policy["install_owner_uid"], expected_path=expected_path, expected_sha256=expected_sha256, label=label, executable=True, ) + runtime_files: list[tuple[ExecutableFileReceipt, int]] = [] try: + for component in runtime_policy["runtime_receipt"]["components"]: + runtime_path = Path(runtime_policy["install_root"]) / "lib" / component["filename"] + runtime_files.append(_approved_file_identity( + runtime_path, + install_root=Path(runtime_policy["install_root"]), + expected_owner_uid=runtime_policy["install_owner_uid"], + expected_path=str(runtime_path), + expected_sha256=component["sha256"], + label=f"{label} runtime component {component['component']}", + executable=False, + )) + verify_approved_executable_identity(path, identity, label=label) + for runtime_identity, _runtime_descriptor in runtime_files: + verify_approved_executable_identity( + Path(runtime_identity.path), + runtime_identity, + label=f"{label} runtime component", + ) + retained_descriptors = (descriptor, *(item[1] for item in runtime_files)) result = subprocess.run( command, executable=f"/proc/self/fd/{descriptor}", - pass_fds=(descriptor,), + pass_fds=retained_descriptors, **kwargs, ) descriptor_after = os.fstat(descriptor) @@ -483,11 +646,260 @@ def run_approved_executable( ): raise TraceError(f"{label} descriptor identity changed during execution") verify_approved_executable_identity(path, identity, label=label) + for runtime_identity, runtime_descriptor in runtime_files: + runtime_after = os.fstat(runtime_descriptor) + if ( + runtime_after.st_dev, + runtime_after.st_ino, + runtime_after.st_uid, + stat.S_IMODE(runtime_after.st_mode), + runtime_after.st_nlink, + runtime_after.st_size, + runtime_after.st_mtime_ns, + runtime_after.st_ctime_ns, + ) != ( + runtime_identity.device, + runtime_identity.inode, + runtime_identity.owner_uid, + runtime_identity.mode, + runtime_identity.link_count, + runtime_identity.byte_count, + runtime_identity.modified_ns, + runtime_identity.changed_ns, + ): + raise TraceError(f"{label} runtime component descriptor changed during execution") + verify_approved_executable_identity( + Path(runtime_identity.path), + runtime_identity, + label=f"{label} runtime component", + ) return result, identity finally: + for _runtime_identity, runtime_descriptor in runtime_files: + os.close(runtime_descriptor) os.close(descriptor) +def install_trust_evidence( + executable: ExecutableFileReceipt, + runtime_files: list[ExecutableFileReceipt], +) -> dict[str, Any]: + files = [executable, *runtime_files] + if any(item.install_root != executable.install_root or item.owner_uid != executable.owner_uid for item in files): + raise TraceError("approved install trust evidence spans multiple roots or owners") + directories: dict[str, tuple[str, int, int, int, int]] = {} + for item in files: + for directory in item.path_chain: + existing = directories.get(directory[0]) + if existing is not None and existing != directory: + raise TraceError("approved install directory identity is inconsistent") + directories[directory[0]] = directory + return { + "format": "dsv41-install-trust", + "version": 1, + "install_root": executable.install_root, + "owner_uid": executable.owner_uid, + "execution_uid": _execution_uid(), + "directories": [ + { + "path": item[0], + "device": item[1], + "inode": item[2], + "owner_uid": item[3], + "mode": item[4], + "effective_write_access": False, + "acl_entries": False, + } + for item in sorted(directories.values()) + ], + "files": [ + { + "path": item.path, + "device": item.device, + "inode": item.inode, + "owner_uid": item.owner_uid, + "mode": item.mode, + "link_count": item.link_count, + "byte_count": item.byte_count, + "modified_ns": item.modified_ns, + "changed_ns": item.changed_ns, + "sha256": item.sha256, + "effective_write_access": False, + "acl_entries": False, + } + for item in sorted(files, key=lambda value: value.path) + ], + } + + +def install_trust_sha256(record: dict[str, Any]) -> str: + validate_install_trust_evidence(record) + return sha256_bytes(canonical_json(record).encode("ascii")) + + +def validate_install_trust_evidence( + record: object, + policy: dict[str, Any] | None = None, +) -> dict[str, Any]: + if not isinstance(record, dict): + raise TraceError("install trust evidence is invalid") + _require_exact_keys( + record, + {"format", "version", "install_root", "owner_uid", "execution_uid", "directories", "files"}, + "install trust evidence", + ) + if record["format"] != "dsv41-install-trust" or record["version"] != 1: + raise TraceError("install trust evidence version is invalid") + install_root = _approval_path(record["install_root"], "install trust root") + owner_uid = record["owner_uid"] + execution_uid = record["execution_uid"] + if type(owner_uid) is not int or owner_uid < 0 or type(execution_uid) is not int or ( + execution_uid <= 0) or execution_uid == owner_uid: + raise TraceError("install trust owner or execution identity is invalid") + directories = record["directories"] + files = record["files"] + if not isinstance(directories, list) or not directories or not isinstance(files, list) or not files: + raise TraceError("install trust evidence is incomplete") + previous_path = None + directory_paths = set() + for directory in directories: + if not isinstance(directory, dict): + raise TraceError("install trust directory evidence is invalid") + _require_exact_keys( + directory, + { + "path", "device", "inode", "owner_uid", "mode", + "effective_write_access", "acl_entries", + }, + "install trust directory evidence", + ) + path = _approval_path(directory["path"], "install trust directory") + if path in directory_paths or (previous_path is not None and path <= previous_path): + raise TraceError("install trust directories are duplicated or unsorted") + directory_paths.add(path) + previous_path = path + if any(type(directory[key]) is not int or directory[key] < 0 for key in ( + "device", "inode", "owner_uid", "mode")) or ( + directory["mode"] & 0o022) or directory["effective_write_access"] is not False or ( + directory["acl_entries"] is not False): + raise TraceError("install trust directory is mutable or malformed") + if directory["owner_uid"] not in {0, owner_uid}: + raise TraceError("install trust directory owner is not trusted") + if (path == install_root or PurePosixPath(install_root) in PurePosixPath(path).parents) and ( + directory["owner_uid"] != owner_uid): + raise TraceError("install trust tree owner differs from approval") + previous_path = None + file_paths = set() + for file_record in files: + if not isinstance(file_record, dict): + raise TraceError("install trust file evidence is invalid") + _require_exact_keys( + file_record, + { + "path", "device", "inode", "owner_uid", "mode", "link_count", "byte_count", + "modified_ns", "changed_ns", "sha256", "effective_write_access", "acl_entries", + }, + "install trust file evidence", + ) + path = _approval_path(file_record["path"], "install trust file") + if path in file_paths or (previous_path is not None and path <= previous_path): + raise TraceError("install trust files are duplicated or unsorted") + file_paths.add(path) + previous_path = path + if any(type(file_record[key]) is not int or file_record[key] < 0 for key in ( + "device", "inode", "owner_uid", "mode", "link_count", "byte_count", + "modified_ns", "changed_ns")) or file_record["owner_uid"] != owner_uid or ( + file_record["mode"] & 0o222) or file_record["link_count"] != 1 or ( + file_record["effective_write_access"] is not False) or file_record["acl_entries"] is not False or ( + re.fullmatch(r"[0-9a-f]{64}", file_record.get("sha256", "")) is None): + raise TraceError("install trust file is mutable or malformed") + if policy is not None: + if install_root != policy["install_root"] or owner_uid != policy["install_owner_uid"]: + raise TraceError("install trust root differs from external approval") + expected_files = { + policy["executable_path"]: policy["executable_sha256"], + **{ + f"{install_root}/lib/{component['filename']}": component["sha256"] + for component in policy["runtime_receipt"]["components"] + }, + } + observed_files = {item["path"]: item["sha256"] for item in files} + if observed_files != expected_files: + raise TraceError("install trust files differ from external approval") + expected_directories = set() + for path in expected_files: + current = PurePosixPath(path).parent + while True: + expected_directories.add(str(current)) + if str(current) == "/": + break + current = current.parent + if directory_paths != expected_directories: + raise TraceError("install trust directory coverage is incomplete") + return record + + +def validate_runtime_build_evidence( + record: object, + policy: dict[str, Any], + *, + label: str, +) -> dict[str, Any]: + if not isinstance(record, dict): + raise TraceError(f"{label} runtime build evidence is invalid") + _require_exact_keys( + record, + { + "revision", + "path", + "sha256", + "runtime_profile", + "runtime_receipt_sha256", + "runtime_libraries", + "runtime_libraries_post", + }, + f"{label} runtime build evidence", + ) + receipt_sha256 = sha256_bytes(canonical_json(policy["runtime_receipt"]).encode("ascii")) + expected = { + "revision": policy["revision"], + "path": policy["executable_path"], + "sha256": policy["executable_sha256"], + "runtime_profile": policy["runtime_profile"], + "runtime_receipt_sha256": receipt_sha256, + } + for key, value in expected.items(): + if record.get(key) != value: + raise TraceError(f"{label} runtime build {key} differs from external approval") + libraries = record["runtime_libraries"] + if libraries != record["runtime_libraries_post"] or not isinstance(libraries, list): + raise TraceError(f"{label} loaded runtime closure changed during execution") + expected_libraries = [] + for component in policy["runtime_receipt"]["components"]: + name = component["component"] + expected_libraries.append({ + "component": name, + "filename": component["filename"], + "path": f"{policy['install_root']}/lib/{component['filename']}", + "sha256": component["sha256"], + "role": { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + }.get(name, f"runtime:{name}"), + "revision": component["revision"], + }) + expected_libraries.sort(key=lambda item: item["path"]) + if libraries != expected_libraries: + raise TraceError(f"{label} loaded runtime libraries differ from external approval") + return record + + +def runtime_build_evidence_sha256(record: object, policy: dict[str, Any], *, label: str) -> str: + validated = validate_runtime_build_evidence(record, policy, label=label) + return sha256_bytes(canonical_json(validated).encode("ascii")) + + def reject_loader_overrides(environment: dict[str, str] | None = None) -> None: values = os.environ if environment is None else environment active = sorted(name for name in FORBIDDEN_LOADER_ENVIRONMENT if values.get(name)) @@ -533,10 +945,18 @@ def _validate_ssh_keygen(path: Path) -> Path: if not path.is_absolute() or path.is_symlink(): raise TraceError("trusted ssh-keygen path must be an absolute non-symlink") try: - mode = path.stat().st_mode + record = path.stat(follow_symlinks=False) except OSError as error: raise TraceError(f"cannot inspect trusted ssh-keygen: {error}") from error - if not stat.S_ISREG(mode) or not os.access(path, os.X_OK): + _immutable_path_chain( + path, + install_root=path.parent.parent, + expected_owner_uid=0, + label="trusted ssh-keygen", + ) + if not stat.S_ISREG(record.st_mode) or record.st_uid != 0 or record.st_nlink != 1 or ( + stat.S_IMODE(record.st_mode) & 0o022) or _path_is_writable_by_execution_identity(path) or ( + _has_access_control_entries(path)) or not (stat.S_IMODE(record.st_mode) & 0o111): raise TraceError("trusted ssh-keygen is not an executable regular file") try: result = subprocess.run( @@ -595,7 +1015,7 @@ def _normalize_public_key(public_key: str, *, allow_comment: bool = False) -> st def _approval_path(value: Any, label: str) -> str: - if not isinstance(value, str) or not value.startswith("/") or ( + if not isinstance(value, str) or not value.startswith("/") or value.startswith("//") or ( ".." in PurePosixPath(value).parts or str(PurePosixPath(value)) != value): raise TraceError(f"{label} is not an absolute canonical path") return value @@ -631,6 +1051,7 @@ def candidate_exporter_approval( "base_revision", "diff_sha256", "install_root", + "install_owner_uid", "executable_path", "executable_sha256", "runtime_profile", @@ -647,6 +1068,8 @@ def candidate_exporter_approval( if re.fullmatch(r"[0-9a-f]{64}", policy.get(key, "")) is None: raise TraceError(f"candidate exporter approval {key} is invalid") install_root = _approval_path(policy["install_root"], "candidate exporter approval install root") + if type(policy["install_owner_uid"]) is not int or policy["install_owner_uid"] < 0: + raise TraceError("candidate exporter approval install owner is invalid") executable_path = _approval_path( policy["executable_path"], "candidate exporter approval executable path") if executable_path != f"{install_root}/bin/llama-deepseek-v41-trace": @@ -712,6 +1135,33 @@ def candidate_exporter_approval( return policy, _approval_digest("candidate-exporter", approval_id, policy) +def validate_tokenizer_policy(record: object) -> dict[str, bool]: + if not isinstance(record, dict): + raise TraceError("tokenizer policy is missing") + _require_exact_keys( + record, + { + "add_bos", + "parse_special", + "detokenize_special", + "remove_leading_bos_before_detokenize", + "require_round_trip", + }, + "tokenizer policy", + ) + if any(type(value) is not bool for value in record.values()): + raise TraceError("tokenizer policy values must be explicit booleans") + if record["parse_special"] is not True or record["detokenize_special"] is not True or ( + record["require_round_trip"] is not True) or ( + record["remove_leading_bos_before_detokenize"] != record["add_bos"]): + raise TraceError("tokenizer policy is not the exact prompt construction policy") + return dict(record) + + +def tokenizer_policy_sha256(record: object) -> str: + return sha256_bytes(canonical_json(validate_tokenizer_policy(record)).encode("ascii")) + + def prompt_builder_approval( approval_id: str, *, @@ -728,12 +1178,14 @@ def prompt_builder_approval( "repository", "revision", "install_root", + "install_owner_uid", "executable_path", "executable_sha256", "source_root", "runtime_receipt", "model_sha256", "corpora", + "tokenizer", "prompts", }, "prompt builder approval", @@ -746,6 +1198,8 @@ def prompt_builder_approval( r"[0-9a-f]{64}", policy.get("executable_sha256", "")) is None: raise TraceError("prompt builder approval executable or model identity is invalid") install_root = _approval_path(policy["install_root"], "prompt builder approval install root") + if type(policy["install_owner_uid"]) is not int or policy["install_owner_uid"] < 0: + raise TraceError("prompt builder approval install owner is invalid") executable_path = _approval_path( policy["executable_path"], "prompt builder approval executable path") source_root = _approval_path(policy["source_root"], "prompt builder approval source root") @@ -809,6 +1263,7 @@ def prompt_builder_approval( raise TraceError("prompt builder approval receipt differs from its runtime profile") if policy["corpora"] != CORPUS_SHA256: raise TraceError("prompt builder approval corpus policy is invalid") + validate_tokenizer_policy(policy["tokenizer"]) prompts = policy["prompts"] if not isinstance(prompts, list) or not prompts: raise TraceError("prompt builder approval prompt policy is empty") @@ -821,7 +1276,7 @@ def prompt_builder_approval( prompt, { "corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", - "prompt_sha256", "prompt_byte_count", "add_bos", + "prompt_sha256", "prompt_byte_count", }, "prompt builder approval prompt record", ) @@ -835,8 +1290,7 @@ def prompt_builder_approval( target_tokens != context - decode_steps): raise TraceError("prompt builder approval prompt configuration is invalid") if re.fullmatch(r"[0-9a-f]{64}", prompt.get("prompt_sha256", "")) is None or ( - type(prompt.get("prompt_byte_count")) is not int or prompt["prompt_byte_count"] <= 0) or ( - type(prompt.get("add_bos")) is not bool): + type(prompt.get("prompt_byte_count")) is not int or prompt["prompt_byte_count"] <= 0): raise TraceError("prompt builder approval prompt output identity is invalid") key = (corpus_name, context, decode_steps) if key in seen_keys or (previous_key is not None and key <= previous_key): @@ -864,9 +1318,25 @@ def approved_prompt_record( return matches[0] -def _read_external_regular_file(path: Path, label: str) -> bytes: +def _read_external_regular_file( + path: Path, + label: str, + *, + trusted_root: Path, + expected_owner_uid: int, + test_only_trust: bool = False, +) -> bytes: if not path.is_absolute() or str(path.resolve()) != str(path): raise TraceError(f"{label} path must be absolute, canonical, and non-symlinked") + if path != trusted_root and trusted_root not in path.parents: + raise TraceError(f"{label} is outside its trusted root") + if not test_only_trust: + _immutable_path_chain( + path, + install_root=trusted_root, + expected_owner_uid=expected_owner_uid, + label=label, + ) try: before = path.stat(follow_symlinks=False) descriptor = os.open( @@ -880,9 +1350,12 @@ def _read_external_regular_file(path: Path, label: str) -> bytes: before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) if identity != ( opened.st_dev, opened.st_ino, opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) or ( - not stat.S_ISREG(opened.st_mode)) or opened.st_nlink != 1: + not stat.S_ISREG(opened.st_mode)) or opened.st_nlink != 1 or ( + not test_only_trust and ( + opened.st_uid != expected_owner_uid or stat.S_IMODE(opened.st_mode) & 0o222 or + _path_is_writable_by_execution_identity(path) or _has_access_control_entries(path))): os.close(descriptor) - raise TraceError(f"{label} must be a regular file with one link") + raise TraceError(f"{label} must be an immutable trusted-owned one-link regular file") try: with os.fdopen(os.dup(descriptor), "rb") as stream: data = stream.read() @@ -907,9 +1380,10 @@ def load_executable_approval_policy( signature_path: Path, *, expected_principal: str, - trusted_approvers: dict[str, str] = APPROVED_EXECUTABLE_APPROVERS, + trusted_approvers: dict[str, dict[str, Any]] = APPROVED_EXECUTABLE_APPROVERS, ssh_keygen: Path | None = None, forbidden_roots: Iterable[Path] = (), + test_only_trust: bool = False, ) -> ExecutableApprovalPolicy: _validate_principal(expected_principal) resolved_policy = policy_path.resolve() @@ -919,12 +1393,29 @@ def load_executable_approval_policy( if resolved_policy == root or root in resolved_policy.parents or ( resolved_signature == root) or root in resolved_signature.parents: raise TraceError("executable approval policy and signature must be outside protected output roots") - public_key = trusted_approvers.get(expected_principal) - if not isinstance(public_key, str): + approver = trusted_approvers.get(expected_principal) + if not isinstance(approver, dict) or set(approver) != { + "public_key", "policy_root", "owner_uid"}: raise TraceError(f"executable approval principal is not trusted: {expected_principal}") - approved_key = _normalize_public_key(public_key) - policy_bytes = _read_external_regular_file(policy_path, "executable approval policy") - signature_bytes = _read_external_regular_file(signature_path, "executable approval signature") + policy_root = Path(_approval_path(approver["policy_root"], "executable approval trusted root")) + owner_uid = approver["owner_uid"] + if type(owner_uid) is not int or owner_uid < 0: + raise TraceError("executable approval trusted owner is invalid") + approved_key = _normalize_public_key(approver["public_key"]) + policy_bytes = _read_external_regular_file( + policy_path, + "executable approval policy", + trusted_root=policy_root, + expected_owner_uid=owner_uid, + test_only_trust=test_only_trust, + ) + signature_bytes = _read_external_regular_file( + signature_path, + "executable approval signature", + trusted_root=policy_root, + expected_owner_uid=owner_uid, + test_only_trust=test_only_trust, + ) try: policy = strict_json_loads(policy_bytes.decode("ascii")) signature = signature_bytes.decode("ascii") @@ -995,12 +1486,18 @@ def load_executable_approval_policy( ) -def approval_binding(kind: str, approval_id: str, digest: str) -> dict[str, str]: +def approval_binding( + kind: str, + approval_id: str, + digest: str, + trust_sha256: str, +) -> dict[str, str]: if kind not in {"candidate_exporter", "prompt_builder"} or re.fullmatch( r"[A-Za-z0-9._-]{1,128}", approval_id) is None or re.fullmatch( - r"[0-9a-f]{64}", digest) is None: + r"[0-9a-f]{64}", digest) is None or re.fullmatch( + r"[0-9a-f]{64}", trust_sha256) is None: raise TraceError("execution approval binding is invalid") - return {"id": approval_id, "sha256": digest} + return {"id": approval_id, "sha256": digest, "install_trust_sha256": trust_sha256} def validate_execution_authorization( @@ -1034,7 +1531,8 @@ def validate_execution_authorization( authorization, { "format", "version", "lane", "challenge", "run_id", "issued_unix", - "expires_unix", "approval_policy_sha256", "verifier_revision", "approvals", + "expires_unix", "approval_policy_sha256", "verifier_revision", + "tokenizer_policy_sha256", "approvals", }, "manifest execution authorization", ) @@ -1073,6 +1571,8 @@ def validate_execution_authorization( raise TraceError("trace signer runtime profile does not match the signed manifest") _prompt_policy, prompt_digest = prompt_builder_approval( expected_prompt_builder_policy_id, policies=prompt_builder_policies) + if authorization.get("tokenizer_policy_sha256") != tokenizer_policy_sha256(_prompt_policy["tokenizer"]): + raise TraceError("manifest tokenizer policy differs from external prompt approval") approvals = authorization["approvals"] required_approvals = {"prompt_builder"} if expected_lane == CANDIDATE_LANE: @@ -1081,16 +1581,22 @@ def validate_execution_authorization( raise TraceError("manifest execution approval bindings are invalid") _require_exact_keys(approvals, required_approvals, "manifest execution approval bindings") prompt_binding = approvals["prompt_builder"] - if prompt_binding != approval_binding( - "prompt_builder", expected_prompt_builder_policy_id, prompt_digest): + if not isinstance(prompt_binding, dict) or set(prompt_binding) != { + "id", "sha256", "install_trust_sha256"} or prompt_binding.get("id") != ( + expected_prompt_builder_policy_id) or prompt_binding.get("sha256") != prompt_digest or re.fullmatch( + r"[0-9a-f]{64}", prompt_binding.get("install_trust_sha256", "")) is None: raise TraceError("manifest prompt builder approval differs from external policy") if expected_lane == CANDIDATE_LANE: if expected_candidate_exporter_policy_id is None: raise TraceError("external candidate exporter approval ID is required") _candidate_policy, candidate_digest = candidate_exporter_approval( expected_candidate_exporter_policy_id, policies=candidate_exporter_policies) - if approvals["candidate_exporter"] != approval_binding( - "candidate_exporter", expected_candidate_exporter_policy_id, candidate_digest): + candidate_binding = approvals["candidate_exporter"] + if not isinstance(candidate_binding, dict) or set(candidate_binding) != { + "id", "sha256", "install_trust_sha256"} or candidate_binding.get("id") != ( + expected_candidate_exporter_policy_id) or candidate_binding.get("sha256") != ( + candidate_digest) or re.fullmatch( + r"[0-9a-f]{64}", candidate_binding.get("install_trust_sha256", "")) is None: raise TraceError("manifest candidate exporter approval differs from external policy") elif expected_candidate_exporter_policy_id is not None: raise TraceError("oracle verification must not specify a candidate exporter approval") @@ -1109,6 +1615,7 @@ def execution_authorization( expires_unix: int, approval_policy_sha256: str, verifier_revision: str, + tokenizer_policy_sha256_value: str, approvals: dict[str, dict[str, str]]) -> dict[str, Any]: if lane not in {CANDIDATE_LANE, ORACLE_LANE}: raise TraceError("execution authorization lane is invalid") @@ -1124,6 +1631,8 @@ def execution_authorization( if re.fullmatch(r"[0-9a-f]{64}", approval_policy_sha256) is None or re.fullmatch( r"[0-9a-f]{40}", verifier_revision) is None: raise TraceError("execution authorization approval policy identity is invalid") + if re.fullmatch(r"[0-9a-f]{64}", tokenizer_policy_sha256_value) is None: + raise TraceError("execution authorization tokenizer policy identity is invalid") required_approvals = {"prompt_builder"} if lane == CANDIDATE_LANE: required_approvals.add("candidate_exporter") @@ -1133,7 +1642,12 @@ def execution_authorization( for kind, binding in approvals.items(): if not isinstance(binding, dict): raise TraceError("execution authorization approval binding is invalid") - if binding != approval_binding(kind, binding.get("id", ""), binding.get("sha256", "")): + if binding != approval_binding( + kind, + binding.get("id", ""), + binding.get("sha256", ""), + binding.get("install_trust_sha256", ""), + ): raise TraceError("execution authorization approval binding is invalid") authorization = { "format": AUTHORIZATION_FORMAT, @@ -1145,6 +1659,7 @@ def execution_authorization( "expires_unix": expires_unix, "approval_policy_sha256": approval_policy_sha256, "verifier_revision": verifier_revision, + "tokenizer_policy_sha256": tokenizer_policy_sha256_value, "approvals": approvals, } return authorization @@ -1482,6 +1997,27 @@ def _bundle_domain( return domain, manifest, events, receipts, contents +def _validate_unsealed_bundle( + root: Path, + manifest: dict[str, Any], + events: list[dict[str, Any]], + receipts: dict[str, BundleFileReceipt], + contents: dict[str, bytes], + verifier: TraceVerifier) -> None: + bundle = object.__new__(_TRACE_BUNDLE_TYPE) + bundle.root = root + bundle.signer_principal = verifier.principal + bundle.verifier = verifier + bundle.manifest = manifest + bundle._file_receipts = receipts + bundle._retained_files = contents + bundle._validate_manifest() + bundle.events = bundle._validate_sealed_events(events, True) + if bundle.manifest.get("event_count") != len(bundle.events): + raise TraceError("manifest event_count mismatch") + bundle._validate_coverage() + + def seal_bundle( root: Path, *, @@ -1511,14 +2047,16 @@ def seal_bundle( signature_path = root / SIGNATURE_NAME if signature_path.exists() or signature_path.is_symlink(): raise TraceError("trace signature envelope already exists") - domain, manifest, _events, _receipts, _contents = _bundle_domain(root) + domain, manifest, events, receipts, contents = _bundle_domain(root) + verification_time = int(time.time()) if verification_unix is None else verification_unix + policy = _signer_policy(trusted_signers, principal) validate_execution_authorization( manifest, - policy=_signer_policy(trusted_signers, principal), + policy=policy, expected_lane=expected_lane, expected_challenge=expected_challenge, expected_run_id=expected_run_id, - verification_unix=int(time.time()) if verification_unix is None else verification_unix, + verification_unix=verification_time, candidate_exporter_policies=candidate_exporter_policies, prompt_builder_policies=prompt_builder_policies, expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, @@ -1526,6 +2064,28 @@ def seal_bundle( expected_approval_policy_sha256=expected_approval_policy_sha256, expected_verifier_revision=expected_verifier_revision, ) + _validate_unsealed_bundle( + root, + manifest, + events, + receipts, + contents, + TraceVerifier( + principal=principal, + trusted_signers=trusted_signers, + ssh_keygen=executable, + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=verification_time, + candidate_exporter_policies=candidate_exporter_policies, + prompt_builder_policies=prompt_builder_policies, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=expected_approval_policy_sha256, + expected_verifier_revision=expected_verifier_revision, + ), + ) try: with tempfile.TemporaryDirectory(prefix="dsv41-trace-sign-") as signing_temp: signing_root = Path(signing_temp).resolve() @@ -2660,17 +3220,40 @@ def _validate_manifest(self) -> None: "builder_sha256": prompt_policy["executable_sha256"], "builder_revision": prompt_policy["revision"], "builder_runtime_profile": prompt_policy["runtime_profile"], + "tokenizer": prompt_policy["tokenizer"], } for key, value in builder_checks.items(): if provenance_record.get(key) != value: raise TraceError(f"prompt provenance {key} differs from external approval") + builder_runtime_build = validate_runtime_build_evidence( + provenance_record.get("builder_runtime_build"), + prompt_policy, + label="prompt builder", + ) + builder_runtime_build_sha256 = runtime_build_evidence_sha256( + builder_runtime_build, + prompt_policy, + label="prompt builder", + ) + if provenance_record.get("builder_runtime_build_sha256") != builder_runtime_build_sha256: + raise TraceError("prompt provenance runtime build SHA-256 mismatch") + builder_trust = validate_install_trust_evidence( + provenance_record.get("builder_install_trust"), prompt_policy) + builder_trust_sha256 = install_trust_sha256(builder_trust) + if provenance_record.get("builder_install_trust_sha256") != builder_trust_sha256: + raise TraceError("prompt provenance install trust SHA-256 mismatch") + if self.manifest["authorization"]["approvals"]["prompt_builder"][ + "install_trust_sha256"] != builder_trust_sha256: + raise TraceError("manifest prompt builder trust differs from signed provenance") for key in ("corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", "prompt_sha256", "prompt_byte_count"): if provenance_record[key] != expected_prompt[key]: raise TraceError(f"prompt provenance {key} differs from approved prompt output") _require_exact_keys( provenance_record, - set(provenance_checks) | set(builder_checks), + set(provenance_checks) | set(builder_checks) | { + "builder_runtime_build", "builder_runtime_build_sha256", + "builder_install_trust", "builder_install_trust_sha256"}, "prompt provenance", ) if self.manifest["prompt"].get("target_tokens") != expected_target: @@ -2692,6 +3275,8 @@ def _validate_manifest(self) -> None: "runtime_receipt_sha256", "exporter_approval_id", "exporter_approval_sha256", + "install_trust", + "install_trust_sha256", }, "llama.cpp candidate attestation", ) @@ -2752,6 +3337,13 @@ def _validate_manifest(self) -> None: for key, value in policy_checks.items(): if candidate_policy[key] != value: raise TraceError(f"candidate {key} differs from external exporter approval") + candidate_trust = validate_install_trust_evidence( + candidate["install_trust"], candidate_policy) + candidate_trust_sha256 = install_trust_sha256(candidate_trust) + if candidate["install_trust_sha256"] != candidate_trust_sha256 or ( + self.manifest["authorization"]["approvals"]["candidate_exporter"][ + "install_trust_sha256"] != candidate_trust_sha256): + raise TraceError("candidate install trust evidence is not bound to authorization") expected_config = { "layer_count": 40, "vocab_size": 129280, @@ -2798,8 +3390,7 @@ def _validate_manifest(self) -> None: "load_mode", "expert_cache_slots", "expert_cache_bytes", - "tokenizer_add_bos", - "tokenizer_parse_special", + "tokenizer", "deepseek41", }, "llama.cpp config", @@ -2819,6 +3410,8 @@ def _validate_manifest(self) -> None: if config.get("kv_type_k") != "f16" or config.get("kv_type_v") != "f16" or ( config.get("flash_attention") is not True) or config.get("load_mode") != 0: raise TraceError("llama.cpp trace inference configuration is invalid") + if validate_tokenizer_policy(config.get("tokenizer")) != prompt_policy["tokenizer"]: + raise TraceError("llama.cpp tokenizer policy differs from external prompt approval") if self.manifest["runtime"] == "ds4": _require_exact_keys( config, @@ -3355,6 +3948,9 @@ def _validate_component_schema(self, event: dict[str, Any]) -> None: raise TraceError("expert.ids contains an out-of-range original expert ID") +_TRACE_BUNDLE_TYPE = TraceBundle + + def first_byte_difference(left: bytes, right: bytes) -> int | None: for index, (a, b) in enumerate(zip(left, right)): if a != b: From 12ee29211d3df2d88c4d6f63394c0bdc84d596cb Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 13:21:38 -0700 Subject: [PATCH 37/56] trace : bind ds4 exporter to immutable approval Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 636 ++++++++++++++++++++++- tools/deepseek-v41-trace/README.md | 17 +- tools/deepseek-v41-trace/run_ds4.py | 303 +++++++++-- tools/deepseek-v41-trace/run_llama.py | 3 + tools/deepseek-v41-trace/trace_format.py | 374 ++++++++++++- 5 files changed, 1244 insertions(+), 89 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 27b3d6394bbd..56ce8a7ce001 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -14,6 +14,7 @@ import time import unittest from argparse import Namespace +from dataclasses import replace from datetime import datetime, timezone from pathlib import Path from unittest import mock @@ -33,8 +34,6 @@ trace.APPROVED_WATCHDOGS[trace.WATCHDOG_SCRIPT_SHA256] = trace.WATCHDOG_REVISION FIXTURE_DS4_EXPORTER_SHA256 = "3" * 64 -trace.APPROVED_EXPORTERS[FIXTURE_DS4_EXPORTER_SHA256] = trace.DS4_REVISION -run_ds4.APPROVED_EXPORTERS[FIXTURE_DS4_EXPORTER_SHA256] = trace.DS4_REVISION TEST_AUTH_ISSUED = int(time.time()) - 60 TEST_AUTH_EXPIRES = TEST_AUTH_ISSUED + 3600 TEST_CHALLENGE = "d" * 64 @@ -43,6 +42,7 @@ "ds4": "apple-ds4-test-run", } TEST_CANDIDATE_EXPORTER_POLICY_ID = "test-candidate-exporter" +TEST_DS4_EXPORTER_POLICY_ID = "test-ds4-exporter" TEST_PROMPT_BUILDER_POLICY_ID = "test-prompt-builder" WATCHDOG_EVENTS = [ @@ -118,6 +118,128 @@ "source": "metal-device-query", } +DS4_RUNTIME_PROFILE = { + "name": "sibling-lib", + "components": ["ds4-runtime", "metal-backend"], + "selected_backend_component": "metal-backend", +} +DS4_RUNTIME_RECEIPT = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": trace.DS4_REVISION, + "profile": "sibling-lib", + "components": [ + { + "component": "ds4-runtime", + "filename": "libds4-runtime.dylib", + "sha256": "a" * 64, + "revision": trace.DS4_REVISION, + }, + { + "component": "metal-backend", + "filename": "libds4-metal.dylib", + "sha256": "b" * 64, + "revision": None, + }, + ], +} +DS4_RUNTIME_LIBRARIES = [ + { + "component": component["component"], + "filename": component["filename"], + "path": f"/Users/oracle/ds4-install/lib/{component['filename']}", + "sha256": component["sha256"], + "role": f"runtime:{component['component']}", + "revision": component["revision"], + } + for component in DS4_RUNTIME_RECEIPT["components"] +] +DS4_RUNTIME_LIBRARIES.sort(key=lambda item: item["path"]) +DS4_RUNTIME_BUILD = { + "revision": trace.DS4_REVISION, + "path": "/Users/oracle/ds4-install/bin/ds4-trace", + "sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": DS4_RUNTIME_PROFILE, + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(DS4_RUNTIME_RECEIPT).encode("ascii")), + "runtime_libraries": DS4_RUNTIME_LIBRARIES, + "runtime_libraries_post": copy.deepcopy(DS4_RUNTIME_LIBRARIES), +} +DS4_INSTALL_TRUST = { + "format": "dsv41-install-trust", + "version": 1, + "install_root": "/Users/oracle/ds4-install", + "owner_uid": 0, + "execution_uid": 501, + "directories": [ + { + "path": path, + "device": 1, + "inode": index, + "owner_uid": 0, + "mode": 0o555, + "effective_write_access": False, + "acl_entries": False, + } + for index, path in enumerate( + ( + "/", + "/Users", + "/Users/oracle", + "/Users/oracle/ds4-install", + "/Users/oracle/ds4-install/bin", + "/Users/oracle/ds4-install/lib", + ), + 1, + ) + ], + "files": [ + { + "path": path, + "device": 1, + "inode": index, + "owner_uid": 0, + "mode": 0o555, + "link_count": 1, + "byte_count": index, + "modified_ns": index, + "changed_ns": index, + "sha256": digest, + "effective_write_access": False, + "acl_entries": False, + } + for index, (path, digest) in enumerate( + sorted(( + ("/Users/oracle/ds4-install/bin/ds4-trace", FIXTURE_DS4_EXPORTER_SHA256), + ("/Users/oracle/ds4-install/lib/libds4-runtime.dylib", "a" * 64), + ("/Users/oracle/ds4-install/lib/libds4-metal.dylib", "b" * 64), + )), + 100, + ) + ], +} +DS4_EXPORTER_POLICY = { + "runtime": "ds4", + "repository": trace.DS4_REPOSITORY, + "revision": trace.DS4_REVISION, + "install_root": "/Users/oracle/ds4-install", + "install_owner_uid": 0, + "executable_path": "/Users/oracle/ds4-install/bin/ds4-trace", + "executable_sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": DS4_RUNTIME_PROFILE, + "runtime_receipt": DS4_RUNTIME_RECEIPT, +} +_DS4_POLICY, DS4_EXPORTER_POLICY_SHA256 = trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: DS4_EXPORTER_POLICY}, +) +DS4_INSTALL_TRUST_SHA256 = trace.install_trust_sha256(DS4_INSTALL_TRUST) +DS4_RUNTIME_BUILD_SHA256 = trace.runtime_build_evidence_sha256( + DS4_RUNTIME_BUILD, + DS4_EXPORTER_POLICY, + label="ds4 exporter", +) + def storage_record(path: str) -> dict[str, object]: model_storage = path.startswith("/mnt/models") @@ -182,7 +304,7 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "temporary_directory": metal_storage_record("/Users/oracle/tmp"), "runner_executable": metal_storage_record("/usr/bin/python3", "/"), "runner_script": metal_storage_record("/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py"), - "exporter": metal_storage_record("/Users/oracle/bin/ds4-trace"), + "exporter": metal_storage_record("/Users/oracle/ds4-install/bin/ds4-trace"), } DS4_HOST_ATTESTATION = { @@ -209,8 +331,16 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "runner_executable_sha256": "1" * 64, "runner_script": "/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py", "runner_script_sha256": "2" * 64, - "exporter_path": "/Users/oracle/bin/ds4-trace", + "exporter_path": "/Users/oracle/ds4-install/bin/ds4-trace", "exporter_sha256": FIXTURE_DS4_EXPORTER_SHA256, + "exporter_approval_id": TEST_DS4_EXPORTER_POLICY_ID, + "exporter_approval_sha256": DS4_EXPORTER_POLICY_SHA256, + "exporter_install_trust_sha256": DS4_INSTALL_TRUST_SHA256, + "exporter_runtime_build_sha256": DS4_RUNTIME_BUILD_SHA256, + "exporter_runtime_profile": DS4_RUNTIME_PROFILE, + "exporter_runtime_receipt_sha256": DS4_RUNTIME_BUILD["runtime_receipt_sha256"], + "producer_revision": trace.DS4_REVISION, + "verifier_revision": "a" * 40, "checkout_path": "/Users/oracle/ds4", "checkout_revision": trace.DS4_REVISION, "command_sha256": "4" * 64, @@ -448,6 +578,21 @@ def materialize_policy_runtime(policy: dict[str, object]) -> None: Path(policy["install_root"]).chmod(0o555) +def materialize_ds4_exporter_policy(root: Path) -> tuple[dict[str, object], Path]: + install = root / "ds4-install" + exporter = install / "bin" / "ds4-trace" + exporter.parent.mkdir(parents=True) + exporter.write_text("#!/bin/sh\nprintf 'approved\\n'\n", encoding="ascii") + exporter.chmod(0o555) + policy = copy.deepcopy(DS4_EXPORTER_POLICY) + policy["install_root"] = str(install) + policy["install_owner_uid"] = os.geteuid() if hasattr(os, "geteuid") else 0 + policy["executable_path"] = str(exporter) + policy["executable_sha256"] = trace.sha256_file(exporter) + materialize_policy_runtime(policy) + return policy, exporter + + def fixture_install_trust(policy: dict[str, object]) -> dict[str, object]: install_root = Path(policy["install_root"]) paths = { @@ -643,8 +788,17 @@ def manifest( { "compiler": "clang", "target": "arm64-apple-darwin", - "path": "/Users/oracle/bin/ds4-trace", + "path": DS4_RUNTIME_BUILD["path"], "sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": copy.deepcopy(DS4_RUNTIME_PROFILE), + "runtime_receipt_sha256": DS4_RUNTIME_BUILD["runtime_receipt_sha256"], + "runtime_libraries": copy.deepcopy(DS4_RUNTIME_LIBRARIES), + "runtime_libraries_post": copy.deepcopy(DS4_RUNTIME_LIBRARIES), + "runtime_module_monitor": { + "mechanism": "dyld-add-image", + "checked_after_trace": True, + "project_additions": [], + }, } if is_ds4 else { @@ -793,6 +947,25 @@ def manifest( } else: result["host"] = dict(DS4_HOST_ATTESTATION) + result["oracle"] = { + "repository": trace.DS4_REPOSITORY, + "revision": trace.DS4_REVISION, + "verifier_revision": "a" * 40, + "executable_path": DS4_RUNTIME_BUILD["path"], + "executable_sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": copy.deepcopy(DS4_RUNTIME_PROFILE), + "runtime_build_sha256": DS4_RUNTIME_BUILD_SHA256, + "runtime_libraries_sha256": trace.sha256_bytes( + trace.canonical_json({ + "pre": DS4_RUNTIME_LIBRARIES, + "post": DS4_RUNTIME_LIBRARIES, + }).encode("ascii")), + "runtime_receipt_sha256": DS4_RUNTIME_BUILD["runtime_receipt_sha256"], + "exporter_approval_id": TEST_DS4_EXPORTER_POLICY_ID, + "exporter_approval_sha256": DS4_EXPORTER_POLICY_SHA256, + "install_trust": copy.deepcopy(DS4_INSTALL_TRUST), + "install_trust_sha256": DS4_INSTALL_TRUST_SHA256, + } result["config"]["prefill_chunk"] = trace.ADMITTED_UBATCH result["config"]["device_backend"] = "Metal" result["config"]["device_registry_id"] = METAL_ACCELERATOR_ATTESTATION["metal_registry_id"] @@ -811,7 +984,14 @@ def manifest( trace.install_trust_sha256(prompt_trust), ), } - if not is_ds4: + if is_ds4: + approvals["ds4_exporter"] = trace.approval_binding( + "ds4_exporter", + TEST_DS4_EXPORTER_POLICY_ID, + DS4_EXPORTER_POLICY_SHA256, + DS4_INSTALL_TRUST_SHA256, + ) + else: candidate_policy = { "runtime": "llama.cpp", "repository": trace.REPOSITORY, @@ -1183,7 +1363,9 @@ def _verifier_for_runtime( }) prompt_policies = {TEST_PROMPT_BUILDER_POLICY_ID: prompt_policy} candidate_policies = {} + ds4_policies = {} candidate_policy_id = None + ds4_policy_id = None if runtime == "llama.cpp": receipt = { "format": "dsv41-runtime-receipt", @@ -1218,6 +1400,10 @@ def _verifier_for_runtime( } candidate_policies[TEST_CANDIDATE_EXPORTER_POLICY_ID] = candidate_policy candidate_policy_id = TEST_CANDIDATE_EXPORTER_POLICY_ID + else: + ds4_policy = copy.deepcopy(DS4_EXPORTER_POLICY) + ds4_policies[TEST_DS4_EXPORTER_POLICY_ID] = ds4_policy + ds4_policy_id = TEST_DS4_EXPORTER_POLICY_ID return trace.TraceVerifier.for_tests( principal, policy["public_key"], @@ -1227,8 +1413,10 @@ def _verifier_for_runtime( expected_challenge=expected_challenge, expected_run_id=expected_run_id or TEST_RUN_IDS[runtime], candidate_exporter_policies=candidate_policies, + ds4_exporter_policies=ds4_policies, prompt_builder_policies=prompt_policies, expected_candidate_exporter_policy_id=candidate_policy_id, + expected_ds4_exporter_policy_id=ds4_policy_id, expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, verification_unix=verification_unix or int(time.time()), ssh_keygen=cls.ssh_keygen, @@ -1268,8 +1456,10 @@ def test_bundle(root: Path, verify_blobs: bool = True, **_kwargs: object) -> obj expected_challenge=authorization["challenge"], expected_run_id=authorization["run_id"], candidate_exporter_policies=verifier.candidate_exporter_policies, + ds4_exporter_policies=verifier.ds4_exporter_policies, prompt_builder_policies=verifier.prompt_builder_policies, expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=verifier.expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, expected_verifier_revision=verifier.expected_verifier_revision, @@ -1311,8 +1501,10 @@ def _seal_test_bundle(self, root: Path) -> str: expected_challenge=authorization["challenge"], expected_run_id=authorization["run_id"], candidate_exporter_policies=verifier.candidate_exporter_policies, + ds4_exporter_policies=verifier.ds4_exporter_policies, prompt_builder_policies=verifier.prompt_builder_policies, expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=verifier.expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, expected_verifier_revision=verifier.expected_verifier_revision, @@ -1461,10 +1653,13 @@ def test_seal_requires_external_trust_and_fixed_verifier(self) -> None: def test_production_executable_approval_maps_fail_closed(self) -> None: self.assertEqual(trace.APPROVED_CANDIDATE_EXPORTERS, {}) + self.assertEqual(trace.APPROVED_DS4_EXPORTERS, {}) self.assertEqual(trace.APPROVED_PROMPT_BUILDERS, {}) self.assertEqual(trace.APPROVED_EXECUTABLE_APPROVERS, {}) with self.assertRaisesRegex(trace.TraceError, "candidate exporter approval is not trusted"): trace.candidate_exporter_approval(TEST_CANDIDATE_EXPORTER_POLICY_ID) + with self.assertRaisesRegex(trace.TraceError, "ds4 exporter approval is not trusted"): + trace.ds4_exporter_approval(TEST_DS4_EXPORTER_POLICY_ID) with self.assertRaisesRegex(trace.TraceError, "prompt builder approval is not trusted"): trace.prompt_builder_approval(TEST_PROMPT_BUILDER_POLICY_ID) @@ -1479,6 +1674,7 @@ def test_external_executable_approval_signature_and_tamper(self) -> None: "verifier_repository": trace.REPOSITORY, "verifier_revision": "a" * 40, "candidate_exporters": verifier.candidate_exporter_policies, + "ds4_exporters": verifier.ds4_exporter_policies, "prompt_builders": verifier.prompt_builder_policies, } with tempfile.TemporaryDirectory() as temp: @@ -1519,6 +1715,7 @@ def test_external_executable_approval_signature_and_tamper(self) -> None: loaded.candidate_exporters, verifier.candidate_exporter_policies, ) + self.assertEqual(loaded.ds4_exporters, verifier.ds4_exporter_policies) with self.assertRaisesRegex(trace.TraceError, "outside protected output roots"): trace.load_executable_approval_policy( policy_path, @@ -1561,6 +1758,7 @@ def test_external_approval_rejects_mutable_root_and_hardlinks(self) -> None: "verifier_repository": trace.REPOSITORY, "verifier_revision": "a" * 40, "candidate_exporters": verifier.candidate_exporter_policies, + "ds4_exporters": verifier.ds4_exporter_policies, "prompt_builders": verifier.prompt_builder_policies, } for mutation, message in ( @@ -1615,6 +1813,90 @@ def test_external_approval_rejects_mutable_root_and_hardlinks(self) -> None: for alias in Path(temp).parent.glob(f"{root.name}-*-alias"): alias.unlink() + def test_external_ds4_approval_requires_distinct_producer_and_verifier(self) -> None: + principal = "dsv41-test-executable-approver" + policy = { + "format": trace.EXECUTABLE_APPROVAL_FORMAT, + "version": trace.EXECUTABLE_APPROVAL_VERSION, + "principal": principal, + "verifier_repository": trace.REPOSITORY, + "verifier_revision": trace.DS4_REVISION, + "candidate_exporters": {}, + "ds4_exporters": { + TEST_DS4_EXPORTER_POLICY_ID: copy.deepcopy(DS4_EXPORTER_POLICY), + }, + "prompt_builders": {}, + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy_path = root / "approval.json" + policy_path.write_text(trace.canonical_json(policy) + "\n", encoding="ascii") + subprocess.run( + [ + str(self.ssh_keygen), + "-Y", "sign", + "-f", str(self.signing_key), + "-n", trace.EXECUTABLE_APPROVAL_NAMESPACE, + str(policy_path), + ], + check=True, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + with self.assertRaisesRegex(trace.TraceError, "producer revision must differ"): + trace.load_executable_approval_policy( + policy_path, + policy_path.with_suffix(".json.sig"), + expected_principal=principal, + trusted_approvers={ + principal: { + "public_key": self.test_signers[self.signer_principal]["public_key"], + "policy_root": str(root), + "owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + }, + }, + ssh_keygen=self.ssh_keygen, + test_only_trust=True, + ) + + def test_ds4_approval_rejects_runtime_identity_mismatches(self) -> None: + for mutation, message in ( + (lambda value: value.update({"runtime": "llama.cpp"}), "runtime identity"), + (lambda value: value.update({"repository": trace.REPOSITORY}), "runtime identity"), + (lambda value: value.update({"revision": "a" * 40}), "runtime identity"), + ( + lambda value: value.update({ + "executable_path": "/Users/oracle/other/bin/ds4-trace", + }), + "outside its install policy", + ), + ( + lambda value: value["runtime_profile"].update({ + "selected_backend_component": "missing", + }), + "runtime profile", + ), + ( + lambda value: value["runtime_receipt"].update({"profile": "co-located"}), + "runtime receipt identity", + ), + ( + lambda value: [ + component.update({"revision": None}) + for component in value["runtime_receipt"]["components"] + ], + "receipt differs|pinned ds4 revision", + ), + ): + with self.subTest(message=message): + policy = copy.deepcopy(DS4_EXPORTER_POLICY) + mutation(policy) + with self.assertRaisesRegex(trace.TraceError, message): + trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + ) + def test_candidate_runner_rejects_unapproved_exporter_before_execution(self) -> None: argv = [ "run_llama.py", @@ -1818,6 +2100,252 @@ def test_writable_install_root_blocks_replace_restore_before_launch(self) -> Non ) execute.assert_not_called() + def test_ds4_exporter_revalidates_each_invocation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + first = run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(first.stdout, "approved\n") + exporter.parent.chmod(0o755) + exporter.chmod(0o755) + exporter.write_text("#!/bin/sh\nprintf 'replacement\\n'\n", encoding="ascii") + exporter.chmod(0o555) + exporter.parent.chmod(0o555) + with mock.patch.object(trace.subprocess, "run") as execute, self.assertRaisesRegex( + run_ds4.TraceError, "SHA-256 differs|identity changed"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + check=True, + capture_output=True, + text=True, + ) + execute.assert_not_called() + + def test_ds4_exporter_detects_swap_after_precheck(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + backup = root / "approved-backup" + replacement = root / "replacement" + replacement.write_text("#!/bin/sh\nprintf 'replacement\\n'\n", encoding="ascii") + replacement.chmod(0o555) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + + def swap_after_precheck(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + exporter.parent.chmod(0o755) + exporter.rename(backup) + replacement.rename(exporter) + exporter.parent.chmod(0o555) + return subprocess.CompletedProcess([str(exporter)], 0, "replacement\n", "") + + with mock.patch.object( + trace.subprocess, "run", side_effect=swap_after_precheck), self.assertRaisesRegex( + run_ds4.TraceError, "descriptor identity changed|SHA-256 differs|identity changed"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + check=True, + capture_output=True, + text=True, + ) + + def test_ds4_exporter_postchecks_failed_invocation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + backup = root / "approved-backup" + replacement = root / "replacement" + replacement.write_text("#!/bin/sh\nexit 7\n", encoding="ascii") + replacement.chmod(0o555) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + + def fail_after_swap(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + exporter.parent.chmod(0o755) + exporter.rename(backup) + replacement.rename(exporter) + exporter.parent.chmod(0o555) + raise subprocess.CalledProcessError(7, [str(exporter)]) + + with mock.patch.object( + trace.subprocess, "run", side_effect=fail_after_swap), self.assertRaisesRegex( + run_ds4.TraceError, "descriptor identity changed|SHA-256 differs|identity changed"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + check=True, + capture_output=True, + text=True, + ) + + def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + Path(policy["install_root"]).chmod(0o777) + try: + with mock.patch.object(trace.subprocess, "run") as execute, self.assertRaisesRegex( + run_ds4.TraceError, "path is mutable|distinct"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + ) + execute.assert_not_called() + finally: + Path(policy["install_root"]).chmod(0o755) + + def test_ds4_exporter_rejects_hardlinks_and_dependency_substitution(self) -> None: + for mutation, message in ( + ("hard-linked-exporter", "one-link"), + ("hard-linked-dependency", "one-link"), + ("dependency-substitution", "SHA-256 differs"), + ("symlinked-exporter", "differs|canonical|aliases"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + library = Path(policy["install_root"]) / "lib" / policy["runtime_receipt"]["components"][0]["filename"] + if mutation == "hard-linked-exporter": + os.link(exporter, root / "exporter-alias") + elif mutation == "hard-linked-dependency": + os.link(library, root / "library-alias") + elif mutation == "dependency-substitution": + library.chmod(0o755) + library.write_bytes(b"substituted") + library.chmod(0o555) + else: + alias = root / "exporter-alias" + alias.symlink_to(exporter) + exporter = alias + with isolated_test_install_trust(), mock.patch.object( + trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace.TraceError, message): + trace.run_approved_executable( + [str(exporter)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + execute.assert_not_called() + + def test_ds4_exporter_rejects_command_path_substitution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + replacement = Path(temp) / "replacement" + replacement.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") + replacement.chmod(0o555) + with isolated_test_install_trust(), mock.patch.object( + trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace.TraceError, "command path differs"): + trace.run_approved_executable( + [str(replacement)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + execute.assert_not_called() + + def test_ds4_exporter_rejects_execution_owned_install(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + with mock.patch.object(trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace.TraceError, "owner must be distinct"): + trace.run_approved_executable( + [str(exporter)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + execute.assert_not_called() + + def test_ds4_policy_and_receipt_mutation_change_approval_identity(self) -> None: + policy = copy.deepcopy(DS4_EXPORTER_POLICY) + _approved, original_sha256 = trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + ) + policy["runtime_receipt"]["components"][0]["sha256"] = "c" * 64 + _mutated, mutated_sha256 = trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + ) + self.assertNotEqual(original_sha256, mutated_sha256) + record = manifest("ds4") + with self.assertRaisesRegex(trace.TraceError, "ds4 exporter approval"): + trace.validate_execution_authorization( + record, + policy=self.test_signers[self.signer_principals["ds4"]], + expected_lane=trace.ORACLE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["ds4"], + candidate_exporter_policies={}, + ds4_exporter_policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + prompt_builder_policies={ + TEST_PROMPT_BUILDER_POLICY_ID: fixture_prompt_builder_policy(b"abc"), + }, + expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=TEST_DS4_EXPORTER_POLICY_ID, + expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, + expected_approval_policy_sha256="e" * 64, + expected_verifier_revision="a" * 40, + verification_unix=TEST_AUTH_ISSUED, + seen_run_ids=None, + ) + def test_install_trust_evidence_rejects_mutability_claims(self) -> None: policy = fixture_prompt_builder_policy(b"prompt") mutations = { @@ -2584,10 +3112,21 @@ def test_metal_accelerator_query_rejects_duplicate_keys(self) -> None: '"backend": "Metal"', '"backend": "Metal", "backend": "Metal"', ) - result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, duplicate, "") - with mock.patch.object(run_ds4.subprocess, "run", return_value=result): + device_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, duplicate, "") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + with mock.patch.object( + run_ds4, "run_exporter_command", + side_effect=[device_result, build_result]) as execute: with self.assertRaisesRegex(preflight.PreflightError, "duplicate JSON key"): - run_ds4.query_accelerator_attestation(Path("/exporter"), "Metal0") + run_ds4.query_accelerator_attestation( + Path("/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 2) def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -3561,7 +4100,7 @@ def test_rejects_unbound_runtime_build_identity(self) -> None: ds4_manifest = manifest("ds4") ds4_manifest["build"]["path"] = "/Users/attacker/unrelated-exporter" - cases.append((ds4_manifest, "ds4 build path")) + cases.append((ds4_manifest, "ds4 exporter runtime build path")) short_revision_manifest = manifest() short_revision_manifest["revision"] = "a" * 9 @@ -4637,6 +5176,7 @@ def test_unapproved_ds4_exporter_is_not_executed(self) -> None: "--corpus-name", "correctness-prose.txt", "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], "--prompt-provenance", str(root / "prompt.json"), + "--ds4-exporter-policy-id", TEST_DS4_EXPORTER_POLICY_ID, "--prompt-builder-policy-id", TEST_PROMPT_BUILDER_POLICY_ID, "--approval-policy", str(root / "approval.json"), "--approval-signature", str(root / "approval.sig"), @@ -4872,30 +5412,74 @@ def test_embeds_rewritten_watchdog_audit_content(self) -> None: ) def test_rejects_unapproved_ds4_exporter(self) -> None: - with self.assertRaisesRegex(preflight.PreflightError, "not approved"): - run_ds4.verify_exporter_approval("a" * 64) + with self.assertRaisesRegex(trace.TraceError, "not trusted"): + trace.ds4_exporter_approval("missing", policies={}) def test_bundle_validation_and_comparison_reject_unapproved_ds4_exporter(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) ds4_root = root / "ds4" - llama_root = root / "llama" with trace.TraceBundleWriter(ds4_root, manifest("ds4")) as writer: add_required_events(writer) - with trace.TraceBundleWriter(llama_root, manifest("llama.cpp")) as writer: + self._seal_test_bundle(ds4_root) + verifier = self._verifier_for_runtime("ds4") + verifier = replace(verifier, ds4_exporter_policies={}) + with self.assertRaisesRegex(trace.TraceError, "not trusted"): + self._trace_bundle_class(ds4_root, verifier=verifier) + + def test_ds4_oracle_trust_bindings_reject_mutation_before_signing(self) -> None: + mutations = ( + ( + lambda value: value["oracle"].update({"exporter_approval_id": "other"}), + "approval differs", + ), + ( + lambda value: value["oracle"].update({"exporter_approval_sha256": "f" * 64}), + "approval differs", + ), + ( + lambda value: value["oracle"].update({"install_trust_sha256": "f" * 64}), + "install trust", + ), + ( + lambda value: value["oracle"].update({"runtime_build_sha256": "f" * 64}), + "runtime build", + ), + ( + lambda value: value["oracle"].update({"runtime_receipt_sha256": "f" * 64}), + "runtime evidence", + ), + ( + lambda value: value["authorization"]["approvals"]["ds4_exporter"].update({ + "install_trust_sha256": "f" * 64, + }), + "install trust", + ), + ) + for mutate, message in mutations: + with self.subTest(message=message), tempfile.TemporaryDirectory() as temp: + record = manifest("ds4") + mutate(record) + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, record) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + self._seal_test_bundle(root) + + def test_ds4_runner_trust_binding_rejects_mutation_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: add_required_events(writer) - approved = dict(trace.APPROVED_EXPORTERS) - try: - trace.APPROVED_EXPORTERS.clear() - with self.assertRaisesRegex(trace.TraceError, "exporter is not approved"): - trace.TraceBundle(ds4_root) - with self.assertRaisesRegex(trace.TraceError, "exporter is not approved"): - trace.command_validate(Namespace(bundle=ds4_root)) - with self.assertRaisesRegex(trace.TraceError, "exporter is not approved"): - trace.command_compare(Namespace(left=ds4_root, right=llama_root, report=None)) - finally: - trace.APPROVED_EXPORTERS.clear() - trace.APPROVED_EXPORTERS.update(approved) + manifest_record = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + original_path = root / manifest_record["audits"]["pre"]["runner"]["path"] + runner = json.loads(json.dumps(DS4_AUDIT_RECORDS["runner"])) + runner["data"]["exporter_install_trust_sha256"] = "f" * 64 + replace_audit_record(root, "pre", "runner", runner) + original_path.unlink() + with self.assertRaisesRegex(trace.TraceError, "install.trust"): + self._seal_test_bundle(root) def test_rejects_preflight_audit_mutation(self) -> None: with tempfile.TemporaryDirectory() as temp: diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index bd06c887ba06..2d49d4cf9f10 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -170,11 +170,13 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM ## Apple Metal oracle execution gate -The external ds4 exporter is not present in the pinned canonical checkout. It remains blocked until a separately built executable is reviewed and its exact SHA-256 is added to the otherwise empty `APPROVED_EXPORTERS` map. Approval is checked before the exporter can run, including device-only preflight, and is checked again whenever a ds4 bundle is validated or compared. `run_ds4.py` requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. +Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. -After the exporter is independently reviewed on an authorized 128 GiB or larger Apple oracle host, add its exact executable SHA-256 and pinned ds4 revision to the shared `APPROVED_EXPORTERS` map in `trace_format.py`; a caller-provided digest alone is not sufficient oracle provenance. The exporter must answer `--dsv41-attest-device Metal0` without loading the model and emit the strict `apple-metal` attestation. Its trace command interface is: +The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. + +After the exporter and its complete runtime receipt are independently reviewed on an authorized 128 GiB or larger Apple oracle host, add the externally reviewed record to a signed executable-approval v2 policy and select it with `--ds4-exporter-policy-id`. Do not add test policy material to the production maps. A caller-provided digest or self-reported build record alone is not sufficient oracle provenance. The exporter must answer `--dsv41-attest-build` and `--dsv41-attest-device Metal0` without loading the model. The device query emits the strict `apple-metal` attestation. Its trace command interface is: The unpublished `ds4gguf` documentation revision `e13893ffcb33e90c8852929303e188102df7a8f5` is provenance only. It is not an executable dependency, exporter approval, or fixture source. Executable tests and fixtures stay in this `strix-llama.cpp` stack. @@ -261,16 +263,17 @@ export TMPDIR=/Users/oracle/dsv41/tmp python3 tools/deepseek-v41-trace/run_ds4.py \ --repo /Users/oracle/src/strix-llama.cpp \ --checkout /Users/oracle/src/ds4-v41 \ - --exporter /Users/oracle/bin/dsv41-trace-exporter \ + --exporter /opt/dsv41/ds4/bin/dsv41-trace-exporter \ --exporter-sha256 \ --model /Users/oracle/models/DeepSeek-V4.1-Flash-Q2.gguf \ --prompt /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt \ --prompt-provenance /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt.provenance.json \ --output /Users/oracle/dsv41/traces/correctness-prose-c32768-ub32 \ --approval-policy "$APPROVAL_POLICY" \ - --approval-policy-signature "$APPROVAL_POLICY_SIGNATURE" \ - --approval-approver-principal "$APPROVAL_APPROVER_PRINCIPAL" \ - --prompt-builder-approval-id "$PROMPT_BUILDER_APPROVAL_ID" \ + --approval-signature "$APPROVAL_POLICY_SIGNATURE" \ + --approval-principal "$APPROVAL_APPROVER_PRINCIPAL" \ + --ds4-exporter-policy-id "$DS4_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ --corpus-name correctness-prose.txt \ --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ --context 32768 \ diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index dab92f874d41..3e268c02d75c 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from typing import Any from preflight import ( PreflightError, @@ -22,11 +23,12 @@ ) from trace_format import ( ADMITTED_UBATCH, - APPROVED_EXPORTERS, APPROVED_PROMPT_BUILDERS, APPROVED_TRACE_SIGNERS, CORPUS_SHA256, + DS4_REPOSITORY, DS4_REVISION, + ExecutableFileReceipt, MODEL_SHA256, NO_EXTERNAL_STATE_STORAGE, ORACLE_LANE, @@ -34,18 +36,28 @@ TraceError, TraceVerifier, approval_binding, + approved_executable_identity, + approved_runtime_file_identities, bind_execution_authorization, canonical_json, + ds4_exporter_approval, execution_authorization, + install_trust_evidence, + install_trust_sha256, load_executable_approval_policy, prompt_builder_approval, reject_loader_overrides, + run_approved_executable, + runtime_build_evidence_sha256, seal_bundle, sha256_bytes, sha256_file, strict_json_loads, tokenizer_policy_sha256, + validate_runtime_build_evidence, validate_signing_identity, + verify_approved_executable_identity, + verify_approved_runtime_file_identities, ) def git_output(checkout: Path, *args: str) -> str: @@ -67,13 +79,6 @@ def verify_checkout(checkout: Path) -> str: return revision -def verify_exporter_approval(exporter_sha256: str) -> None: - if APPROVED_EXPORTERS.get(exporter_sha256) != DS4_REVISION: - raise PreflightError( - "ds4 trace exporter is not approved for the pinned ds4 revision; " - "publish and review the exporter before cross-runtime execution") - - def validate_accelerator_attestation( record: object, *, @@ -122,30 +127,103 @@ def validate_accelerator_attestation( return dict(record) -def query_accelerator_attestation(exporter: Path, device: str) -> dict[str, object]: - try: - result = subprocess.run( - [str(exporter), "--dsv41-attest-device", device], - check=False, - capture_output=True, - text=True, - ) - except OSError as error: - raise PreflightError(f"cannot query selected accelerator: {error}") from error +def run_exporter_command( + command: list[str], + *, + exporter: Path, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any], + **kwargs: Any) -> subprocess.CompletedProcess[Any]: + result, executed_identity = run_approved_executable( + command, + path=exporter, + runtime_policy=exporter_policy, + expected_path=exporter_policy["executable_path"], + expected_sha256=exporter_policy["executable_sha256"], + label="ds4 exporter", + **kwargs, + ) + if executed_identity != exporter_identity: + raise PreflightError("ds4 exporter execution identity differs from external approval") + return result + + +def query_runtime_build_attestation( + exporter: Path, + *, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any]) -> dict[str, Any]: + result = run_exporter_command( + [str(exporter), "--dsv41-attest-build"], + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + check=False, + capture_output=True, + text=True, + ) if result.returncode != 0: - detail = result.stderr.strip() or f"exit {result.returncode}" - raise PreflightError(f"selected accelerator query failed: {detail}") + raise PreflightError(f"ds4 exporter build attestation failed: {result.stderr.strip()}") try: record = strict_json_loads(result.stdout) + return validate_runtime_build_evidence(record, exporter_policy, label="ds4 exporter") except TraceError as error: - raise PreflightError(f"selected accelerator query returned invalid JSON: {error}") from error - return validate_accelerator_attestation(record, expected_device=device) + raise PreflightError(f"ds4 exporter build attestation is invalid: {error}") from error + + +def query_accelerator_attestation( + exporter: Path, + device: str, + *, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any], + expected_runtime_build: dict[str, Any]) -> dict[str, object]: + result = run_exporter_command( + [str(exporter), "--dsv41-attest-device", device], + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + check=False, + capture_output=True, + text=True, + ) + validation_error = None + attestation = None + if result.returncode != 0: + detail = result.stderr.strip() or f"exit {result.returncode}" + validation_error = PreflightError(f"selected accelerator query failed: {detail}") + else: + try: + record = strict_json_loads(result.stdout) + attestation = validate_accelerator_attestation(record, expected_device=device) + except (TraceError, PreflightError) as error: + validation_error = PreflightError( + f"selected accelerator query returned invalid attestation: {error}") + post_runtime_build = query_runtime_build_attestation( + exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + ) + if post_runtime_build != expected_runtime_build: + raise PreflightError("ds4 exporter build identity changed during accelerator query") + if validation_error is not None: + raise validation_error + if attestation is None: + raise PreflightError("selected accelerator attestation is missing") + return attestation def runner_attestation( *, exporter: Path, exporter_sha256: str, + exporter_approval_id: str, + exporter_approval_sha256: str, + exporter_install_trust_sha256: str, + exporter_runtime_build_sha256: str, + exporter_runtime_profile: dict[str, object], + exporter_runtime_receipt_sha256: str, + verifier_revision: str, checkout: Path, command: list[str]) -> dict[str, object]: runner_executable = resolved(Path(sys.executable)) @@ -164,6 +242,14 @@ def runner_attestation( "runner_script_sha256": sha256_file(runner_script), "exporter_path": str(exporter), "exporter_sha256": exporter_sha256, + "exporter_approval_id": exporter_approval_id, + "exporter_approval_sha256": exporter_approval_sha256, + "exporter_install_trust_sha256": exporter_install_trust_sha256, + "exporter_runtime_build_sha256": exporter_runtime_build_sha256, + "exporter_runtime_profile": exporter_runtime_profile, + "exporter_runtime_receipt_sha256": exporter_runtime_receipt_sha256, + "producer_revision": DS4_REVISION, + "verifier_revision": verifier_revision, "checkout_path": str(checkout), "checkout_revision": DS4_REVISION, "command_sha256": sha256_bytes(canonical_json(command).encode("ascii")), @@ -174,7 +260,14 @@ def bind_oracle_attestation( output: Path, audit: dict[str, object], accelerator: dict[str, object], - command: list[str]) -> None: + command: list[str], + *, + exporter_policy: dict[str, object], + exporter_approval_id: str, + exporter_approval_sha256: str, + exporter_install_trust: dict[str, object], + runtime_build: dict[str, object], + verifier_revision: str) -> None: manifest_path = output / "manifest.json" try: manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) @@ -203,6 +296,46 @@ def bind_oracle_attestation( runner = audit.get("runner") if not isinstance(runner, dict) or build.get("sha256") != runner.get("exporter_sha256"): raise PreflightError("ds4 trace build SHA-256 differs from the executed exporter") + build_evidence = { + "revision": manifest.get("revision"), + "path": build.get("path"), + "sha256": build.get("sha256"), + "runtime_profile": build.get("runtime_profile"), + "runtime_receipt_sha256": build.get("runtime_receipt_sha256"), + "runtime_libraries": build.get("runtime_libraries"), + "runtime_libraries_post": build.get("runtime_libraries_post"), + } + try: + validated_build = validate_runtime_build_evidence( + build_evidence, exporter_policy, label="ds4 exporter") + except TraceError as error: + raise PreflightError(f"ds4 trace runtime build differs from external approval: {error}") from error + if validated_build != runtime_build: + raise PreflightError("ds4 trace runtime build differs from measured exporter attestation") + runtime_build_sha256 = runtime_build_evidence_sha256( + validated_build, exporter_policy, label="ds4 exporter") + runtime_libraries_sha256 = sha256_bytes(canonical_json({ + "pre": validated_build["runtime_libraries"], + "post": validated_build["runtime_libraries_post"], + }).encode("ascii")) + runtime_receipt_sha256 = sha256_bytes( + canonical_json(exporter_policy["runtime_receipt"]).encode("ascii")) + trust_sha256 = install_trust_sha256(exporter_install_trust) + manifest["oracle"] = { + "repository": DS4_REPOSITORY, + "revision": DS4_REVISION, + "verifier_revision": verifier_revision, + "executable_path": exporter_policy["executable_path"], + "executable_sha256": exporter_policy["executable_sha256"], + "runtime_profile": exporter_policy["runtime_profile"], + "runtime_build_sha256": runtime_build_sha256, + "runtime_libraries_sha256": runtime_libraries_sha256, + "runtime_receipt_sha256": runtime_receipt_sha256, + "exporter_approval_id": exporter_approval_id, + "exporter_approval_sha256": exporter_approval_sha256, + "install_trust": exporter_install_trust, + "install_trust_sha256": trust_sha256, + } storage_policy = audit.get("storage_policy") if storage_policy != NO_EXTERNAL_STATE_STORAGE: raise PreflightError("ds4 external cache/state storage policy is invalid") @@ -290,6 +423,7 @@ def main() -> int: parser.add_argument("--run-id", required=True) parser.add_argument("--authorization-issued-unix", type=int, required=True) parser.add_argument("--authorization-expires-unix", type=int, required=True) + parser.add_argument("--ds4-exporter-policy-id", required=True) parser.add_argument("--prompt-builder-policy-id", required=True) parser.add_argument("--approval-policy", type=Path, required=True) parser.add_argument("--approval-signature", type=Path, required=True) @@ -314,6 +448,10 @@ def main() -> int: args.prompt_builder_policy_id, policies=approval_policy.prompt_builders, ) + exporter_policy, exporter_policy_sha256 = ds4_exporter_approval( + args.ds4_exporter_policy_id, + policies=approval_policy.ds4_exporters, + ) harness_repo = resolved(args.repo) harness_revision = subprocess.check_output( ["git", "-C", str(harness_repo), "rev-parse", "HEAD"], @@ -321,6 +459,39 @@ def main() -> int: ).decode("ascii").strip() if harness_revision != approval_policy.verifier_revision: raise PreflightError("ds4 verifier checkout differs from the external approval policy") + checkout = resolved(args.checkout) + checkout_revision = verify_checkout(checkout) + if checkout_revision != DS4_REVISION: + raise PreflightError( + f"ds4 revision mismatch: expected {DS4_REVISION}, found {checkout_revision}") + if not args.exporter.is_absolute() or str(args.exporter) != exporter_policy["executable_path"]: + raise PreflightError("ds4 exporter path or caller digest differs from external approval") + exporter = resolved(args.exporter) + if str(exporter) != str(args.exporter) or ( + args.exporter_sha256 != exporter_policy["executable_sha256"]): + raise PreflightError("ds4 exporter path or caller digest differs from external approval") + exporter_identity = approved_executable_identity( + exporter, + install_root=exporter_policy["install_root"], + expected_owner_uid=exporter_policy["install_owner_uid"], + expected_path=exporter_policy["executable_path"], + expected_sha256=exporter_policy["executable_sha256"], + label="ds4 exporter", + ) + runtime_identities = approved_runtime_file_identities( + exporter_policy, label="ds4 exporter") + exporter_install_trust = install_trust_evidence( + exporter_identity, runtime_identities) + exporter_install_trust_sha256 = install_trust_sha256(exporter_install_trust) + pre_runtime_build = query_runtime_build_attestation( + exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + ) + runtime_build_sha256 = runtime_build_evidence_sha256( + pre_runtime_build, exporter_policy, label="ds4 exporter") + runtime_receipt_sha256 = sha256_bytes( + canonical_json(exporter_policy["runtime_receipt"]).encode("ascii")) if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") model_sha256 = sha256_file(resolved(args.model)) @@ -351,6 +522,12 @@ def main() -> int: verifier_revision=approval_policy.verifier_revision, tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), approvals={ + "ds4_exporter": approval_binding( + "ds4_exporter", + args.ds4_exporter_policy_id, + exporter_policy_sha256, + exporter_install_trust_sha256, + ), "prompt_builder": approval_binding( "prompt_builder", args.prompt_builder_policy_id, @@ -359,14 +536,7 @@ def main() -> int: ), }, ) - exporter = resolved(args.exporter) - if not exporter.is_file() or not os.access(exporter, os.X_OK): - raise PreflightError(f"trace exporter is not executable: {exporter}") - exporter_sha256 = sha256_file(exporter) - if exporter_sha256 != args.exporter_sha256: - raise PreflightError( - f"trace exporter SHA-256 mismatch: expected {args.exporter_sha256}, found {exporter_sha256}") - verify_exporter_approval(exporter_sha256) + exporter_sha256 = exporter_identity.sha256 validate_signing_identity( args.signing_key, args.signer_principal, @@ -383,11 +553,24 @@ def main() -> int: "--prefill-chunk", str(args.prefill_chunk), "--device", args.device, ] - accelerator = query_accelerator_attestation(exporter, args.device) + accelerator = query_accelerator_attestation( + exporter, + args.device, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + expected_runtime_build=pre_runtime_build, + ) runner = runner_attestation( exporter=exporter, exporter_sha256=exporter_sha256, - checkout=resolved(args.checkout), + exporter_approval_id=args.ds4_exporter_policy_id, + exporter_approval_sha256=exporter_policy_sha256, + exporter_install_trust_sha256=exporter_install_trust_sha256, + exporter_runtime_build_sha256=runtime_build_sha256, + exporter_runtime_profile=exporter_policy["runtime_profile"], + exporter_runtime_receipt_sha256=runtime_receipt_sha256, + verifier_revision=approval_policy.verifier_revision, + checkout=checkout, command=command, ) if args.preflight_only: @@ -398,15 +581,47 @@ def main() -> int: if output.exists() and any(output.iterdir()): raise PreflightError(f"trace output directory is not empty: {output}") preflight_audit = preflight(args, accelerator=accelerator, runner=runner) - preflight_audit["exporter"] = {"path": str(exporter), "sha256": exporter_sha256} + preflight_audit["exporter"] = { + "path": str(exporter), + "sha256": exporter_sha256, + "approval_id": args.ds4_exporter_policy_id, + "approval_sha256": exporter_policy_sha256, + "install_trust_sha256": exporter_install_trust_sha256, + "runtime_build_sha256": runtime_build_sha256, + "runtime_receipt_sha256": runtime_receipt_sha256, + } pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) pre_audit_digests = seal_audits(pre_audits) print("exec:", shlex.join(command), file=sys.stderr) - result = subprocess.run(command, cwd=resolved(args.checkout), check=False) + result = run_exporter_command( + command, + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + cwd=checkout, + check=False, + ) + verify_approved_executable_identity( + exporter, exporter_identity, label="ds4 exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="ds4 exporter") + post_trace_runtime_build = query_runtime_build_attestation( + exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + ) + if post_trace_runtime_build != pre_runtime_build: + raise PreflightError("ds4 exporter build identity changed during trace execution") if result.returncode != 0: return result.returncode verify_sealed_audits(pre_audits, pre_audit_digests) - post_accelerator = query_accelerator_attestation(exporter, args.device) + post_accelerator = query_accelerator_attestation( + exporter, + args.device, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + expected_runtime_build=pre_runtime_build, + ) if post_accelerator != accelerator: raise PreflightError("selected accelerator identity changed during trace execution") postflight_audit = preflight(args, accelerator=post_accelerator, runner=runner) @@ -415,7 +630,18 @@ def main() -> int: post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) bind_prompt_provenance(output, provenance) - bind_oracle_attestation(output, preflight_audit, accelerator, command) + bind_oracle_attestation( + output, + preflight_audit, + accelerator, + command, + exporter_policy=exporter_policy, + exporter_approval_id=args.ds4_exporter_policy_id, + exporter_approval_sha256=exporter_policy_sha256, + exporter_install_trust=exporter_install_trust, + runtime_build=pre_runtime_build, + verifier_revision=approval_policy.verifier_revision, + ) bind_execution_authorization(output, authorization) seal_bundle( output, @@ -425,8 +651,10 @@ def main() -> int: expected_challenge=args.execution_challenge, expected_run_id=args.run_id, candidate_exporter_policies={}, + ds4_exporter_policies=approval_policy.ds4_exporters, prompt_builder_policies=approval_policy.prompt_builders, expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=args.ds4_exporter_policy_id, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, expected_approval_policy_sha256=approval_policy.sha256, expected_verifier_revision=approval_policy.verifier_revision, @@ -440,6 +668,7 @@ def main() -> int: expected_challenge=args.execution_challenge, expected_run_id=args.run_id, expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=args.ds4_exporter_policy_id, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 2736715d318b..2cc8e4193c07 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -737,8 +737,10 @@ def main() -> int: expected_challenge=args.execution_challenge, expected_run_id=args.run_id, candidate_exporter_policies=approval_policy.candidate_exporters, + ds4_exporter_policies={}, prompt_builder_policies=approval_policy.prompt_builders, expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, expected_approval_policy_sha256=approval_policy.sha256, expected_verifier_revision=approval_policy.verifier_revision, @@ -752,6 +754,7 @@ def main() -> int: expected_challenge=args.execution_challenge, expected_run_id=args.run_id, expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 0933b0efdc4f..11e7d7d23e49 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -19,10 +19,11 @@ TRACE_FORMAT = "dsv41-trace" TRACE_VERSION = 2 DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" -APPROVED_EXPORTERS: dict[str, str] = {} +DS4_REPOSITORY = "antirez/ds4" APPROVED_TRACE_SIGNERS: dict[str, dict[str, str]] = {} APPROVED_EXECUTABLE_APPROVERS: dict[str, dict[str, Any]] = {} APPROVED_CANDIDATE_EXPORTERS: dict[str, dict[str, Any]] = {} +APPROVED_DS4_EXPORTERS: dict[str, dict[str, Any]] = {} APPROVED_PROMPT_BUILDERS: dict[str, dict[str, Any]] = {} EXECUTABLE_APPROVAL_FORMAT = "dsv41-executable-approval" EXECUTABLE_APPROVAL_VERSION = 2 @@ -143,8 +144,10 @@ class TraceVerifier: expected_run_id: str verification_unix: int candidate_exporter_policies: dict[str, dict[str, Any]] + ds4_exporter_policies: dict[str, dict[str, Any]] prompt_builder_policies: dict[str, dict[str, Any]] expected_candidate_exporter_policy_id: str | None + expected_ds4_exporter_policy_id: str | None expected_prompt_builder_policy_id: str expected_approval_policy_sha256: str expected_verifier_revision: str @@ -160,6 +163,7 @@ def production( expected_challenge: str, expected_run_id: str, expected_candidate_exporter_policy_id: str | None, + expected_ds4_exporter_policy_id: str | None, expected_prompt_builder_policy_id: str, approval_policy: "ExecutableApprovalPolicy | None" = None, verification_unix: int | None, @@ -175,10 +179,14 @@ def production( candidate_exporter_policies=( approval_policy.candidate_exporters if approval_policy is not None else APPROVED_CANDIDATE_EXPORTERS), + ds4_exporter_policies=( + approval_policy.ds4_exporters + if approval_policy is not None else APPROVED_DS4_EXPORTERS), prompt_builder_policies=( approval_policy.prompt_builders if approval_policy is not None else APPROVED_PROMPT_BUILDERS), expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, expected_approval_policy_sha256=( approval_policy.sha256 if approval_policy is not None else ""), @@ -199,8 +207,10 @@ def for_tests( expected_challenge: str, expected_run_id: str, candidate_exporter_policies: dict[str, dict[str, Any]] | None = None, + ds4_exporter_policies: dict[str, dict[str, Any]] | None = None, prompt_builder_policies: dict[str, dict[str, Any]] | None = None, expected_candidate_exporter_policy_id: str | None = None, + expected_ds4_exporter_policy_id: str | None = None, expected_prompt_builder_policy_id: str = "", expected_approval_policy_sha256: str = "e" * 64, expected_verifier_revision: str = "a" * 40, @@ -223,8 +233,10 @@ def for_tests( expected_run_id=expected_run_id, verification_unix=verification_unix, candidate_exporter_policies=candidate_exporter_policies or {}, + ds4_exporter_policies=ds4_exporter_policies or {}, prompt_builder_policies=prompt_builder_policies or {}, expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, expected_approval_policy_sha256=expected_approval_policy_sha256, expected_verifier_revision=expected_verifier_revision, @@ -264,6 +276,7 @@ class ExecutableApprovalPolicy: principal: str verifier_revision: str candidate_exporters: dict[str, dict[str, Any]] + ds4_exporters: dict[str, dict[str, Any]] prompt_builders: dict[str, dict[str, Any]] sha256: str @@ -592,8 +605,12 @@ def run_approved_executable( label: str, **kwargs: Any, ) -> tuple[subprocess.CompletedProcess[Any], ExecutableFileReceipt]: - if sys.platform != "linux": - raise TraceError(f"{label} descriptor execution requires Linux") + if sys.platform not in {"linux", "darwin"}: + raise TraceError(f"{label} immutable execution is unsupported on this platform") + if not command or command[0] != str(path): + raise TraceError(f"{label} command path differs from external approval") + if "executable" in kwargs or "pass_fds" in kwargs: + raise TraceError(f"{label} execution parameters may not override immutable launch controls") identity, descriptor = _approved_file_identity( path, install_root=Path(runtime_policy["install_root"]), @@ -624,12 +641,18 @@ def run_approved_executable( label=f"{label} runtime component", ) retained_descriptors = (descriptor, *(item[1] for item in runtime_files)) - result = subprocess.run( - command, - executable=f"/proc/self/fd/{descriptor}", - pass_fds=retained_descriptors, + launch = { + "pass_fds": retained_descriptors, **kwargs, - ) + } + if sys.platform == "linux": + launch["executable"] = f"/proc/self/fd/{descriptor}" + execution_error: OSError | subprocess.SubprocessError | None = None + result = None + try: + result = subprocess.run(command, **launch) + except (OSError, subprocess.SubprocessError) as error: + execution_error = error descriptor_after = os.fstat(descriptor) if ( descriptor_after.st_dev, @@ -673,6 +696,10 @@ def run_approved_executable( runtime_identity, label=f"{label} runtime component", ) + if execution_error is not None: + raise execution_error + if result is None: + raise TraceError(f"{label} execution did not return a result") return result, identity finally: for _runtime_identity, runtime_descriptor in runtime_files: @@ -1135,6 +1162,101 @@ def candidate_exporter_approval( return policy, _approval_digest("candidate-exporter", approval_id, policy) +def ds4_exporter_approval( + approval_id: str, + *, + policies: dict[str, dict[str, Any]] = APPROVED_DS4_EXPORTERS, +) -> tuple[dict[str, Any], str]: + policy = policies.get(approval_id) + if not isinstance(policy, dict): + raise TraceError(f"ds4 exporter approval is not trusted: {approval_id}") + _require_exact_keys( + policy, + { + "runtime", + "repository", + "revision", + "install_root", + "install_owner_uid", + "executable_path", + "executable_sha256", + "runtime_profile", + "runtime_receipt", + }, + "ds4 exporter approval", + ) + if policy["runtime"] != "ds4" or policy["repository"] != DS4_REPOSITORY or ( + policy["revision"] != DS4_REVISION): + raise TraceError("ds4 exporter approval runtime identity is invalid") + if re.fullmatch(r"[0-9a-f]{64}", policy.get("executable_sha256", "")) is None: + raise TraceError("ds4 exporter approval executable SHA-256 is invalid") + install_root = _approval_path(policy["install_root"], "ds4 exporter approval install root") + if type(policy["install_owner_uid"]) is not int or policy["install_owner_uid"] < 0: + raise TraceError("ds4 exporter approval install owner is invalid") + executable_path = _approval_path( + policy["executable_path"], "ds4 exporter approval executable path") + executable = PurePosixPath(executable_path) + if executable.parent != PurePosixPath(install_root) / "bin": + raise TraceError("ds4 exporter approval executable path is outside its install policy") + profile = policy["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("ds4 exporter approval runtime profile is invalid") + _require_exact_keys( + profile, {"name", "components", "selected_backend_component"}, + "ds4 exporter approval runtime profile") + components = profile["components"] + selected_backend = profile["selected_backend_component"] + if profile["name"] not in {"co-located", "sibling-lib"} or not isinstance(components, list) or ( + components != sorted(components)) or len(components) != len(set(components)) or not components or any( + not isinstance(component, str) or re.fullmatch(r"[a-z0-9-]+", component) is None + for component in components) or not isinstance(selected_backend, str) or ( + selected_backend not in components): + raise TraceError("ds4 exporter approval runtime profile is invalid") + receipt = policy["runtime_receipt"] + if not isinstance(receipt, dict): + raise TraceError("ds4 exporter approval runtime receipt is invalid") + _require_exact_keys( + receipt, {"format", "version", "revision", "profile", "components"}, + "ds4 exporter approval runtime receipt") + if receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or ( + receipt["revision"] != DS4_REVISION) or receipt["profile"] != profile["name"]: + raise TraceError("ds4 exporter approval runtime receipt identity is invalid") + receipt_components = receipt["components"] + if not isinstance(receipt_components, list) or receipt_components != sorted( + receipt_components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + raise TraceError("ds4 exporter approval runtime receipt components are not canonical") + seen_components = set() + seen_filenames = set() + seen_digests = set() + revision_bearing = 0 + for component in receipt_components: + if not isinstance(component, dict): + raise TraceError("ds4 exporter approval runtime receipt component is invalid") + _require_exact_keys( + component, {"component", "filename", "sha256", "revision"}, + "ds4 exporter approval runtime receipt component") + name = component["component"] + filename = component["filename"] + digest = component["sha256"] + revision = component["revision"] + if name not in components or name in seen_components: + raise TraceError("ds4 exporter approval runtime receipt component name is invalid") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None or ( + filename in seen_filenames): + raise TraceError("ds4 exporter approval runtime receipt filename is invalid") + if re.fullmatch(r"[0-9a-f]{64}", digest or "") is None or digest in seen_digests: + raise TraceError("ds4 exporter approval runtime receipt digest is invalid") + if revision not in {None, DS4_REVISION}: + raise TraceError("ds4 exporter approval runtime receipt revision is invalid") + revision_bearing += revision == DS4_REVISION + seen_components.add(name) + seen_filenames.add(filename) + seen_digests.add(digest) + if seen_components != set(components) or revision_bearing == 0: + raise TraceError("ds4 exporter approval receipt differs from its runtime profile") + return policy, _approval_digest("ds4-exporter", approval_id, policy) + + def validate_tokenizer_policy(record: object) -> dict[str, bool]: if not isinstance(record, dict): raise TraceError("tokenizer policy is missing") @@ -1429,7 +1551,7 @@ def load_executable_approval_policy( policy, { "format", "version", "principal", "verifier_repository", "verifier_revision", - "candidate_exporters", "prompt_builders", + "candidate_exporters", "ds4_exporters", "prompt_builders", }, "executable approval policy", ) @@ -1440,11 +1562,17 @@ def load_executable_approval_policy( r"[0-9a-f]{40}", policy.get("verifier_revision", "")) is None: raise TraceError("executable approval policy identity is invalid") candidate_exporters = policy["candidate_exporters"] + ds4_exporters = policy["ds4_exporters"] prompt_builders = policy["prompt_builders"] - if not isinstance(candidate_exporters, dict) or not isinstance(prompt_builders, dict): + if not isinstance(candidate_exporters, dict) or not isinstance(ds4_exporters, dict) or ( + not isinstance(prompt_builders, dict)): raise TraceError("executable approval policy maps are invalid") for approval_id in sorted(candidate_exporters): candidate_exporter_approval(approval_id, policies=candidate_exporters) + for approval_id in sorted(ds4_exporters): + ds4_policy, _digest = ds4_exporter_approval(approval_id, policies=ds4_exporters) + if ds4_policy["revision"] == policy["verifier_revision"]: + raise TraceError("ds4 exporter producer revision must differ from the verifier revision") for approval_id in sorted(prompt_builders): prompt_builder_approval(approval_id, policies=prompt_builders) if not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or not signature.endswith( @@ -1481,6 +1609,7 @@ def load_executable_approval_policy( principal=expected_principal, verifier_revision=policy["verifier_revision"], candidate_exporters=candidate_exporters, + ds4_exporters=ds4_exporters, prompt_builders=prompt_builders, sha256=sha256_bytes(policy_bytes), ) @@ -1492,7 +1621,7 @@ def approval_binding( digest: str, trust_sha256: str, ) -> dict[str, str]: - if kind not in {"candidate_exporter", "prompt_builder"} or re.fullmatch( + if kind not in {"candidate_exporter", "ds4_exporter", "prompt_builder"} or re.fullmatch( r"[A-Za-z0-9._-]{1,128}", approval_id) is None or re.fullmatch( r"[0-9a-f]{64}", digest) is None or re.fullmatch( r"[0-9a-f]{64}", trust_sha256) is None: @@ -1509,8 +1638,10 @@ def validate_execution_authorization( expected_run_id: str, verification_unix: int, candidate_exporter_policies: dict[str, dict[str, Any]], + ds4_exporter_policies: dict[str, dict[str, Any]], prompt_builder_policies: dict[str, dict[str, Any]], expected_candidate_exporter_policy_id: str | None, + expected_ds4_exporter_policy_id: str | None, expected_prompt_builder_policy_id: str, expected_approval_policy_sha256: str, expected_verifier_revision: str, @@ -1577,6 +1708,8 @@ def validate_execution_authorization( required_approvals = {"prompt_builder"} if expected_lane == CANDIDATE_LANE: required_approvals.add("candidate_exporter") + else: + required_approvals.add("ds4_exporter") if not isinstance(approvals, dict): raise TraceError("manifest execution approval bindings are invalid") _require_exact_keys(approvals, required_approvals, "manifest execution approval bindings") @@ -1589,6 +1722,8 @@ def validate_execution_authorization( if expected_lane == CANDIDATE_LANE: if expected_candidate_exporter_policy_id is None: raise TraceError("external candidate exporter approval ID is required") + if expected_ds4_exporter_policy_id is not None: + raise TraceError("candidate verification must not specify a ds4 exporter approval") _candidate_policy, candidate_digest = candidate_exporter_approval( expected_candidate_exporter_policy_id, policies=candidate_exporter_policies) candidate_binding = approvals["candidate_exporter"] @@ -1598,8 +1733,20 @@ def validate_execution_authorization( candidate_digest) or re.fullmatch( r"[0-9a-f]{64}", candidate_binding.get("install_trust_sha256", "")) is None: raise TraceError("manifest candidate exporter approval differs from external policy") - elif expected_candidate_exporter_policy_id is not None: - raise TraceError("oracle verification must not specify a candidate exporter approval") + else: + if expected_candidate_exporter_policy_id is not None: + raise TraceError("oracle verification must not specify a candidate exporter approval") + if expected_ds4_exporter_policy_id is None: + raise TraceError("external ds4 exporter approval ID is required") + _ds4_policy, ds4_digest = ds4_exporter_approval( + expected_ds4_exporter_policy_id, policies=ds4_exporter_policies) + ds4_binding = approvals["ds4_exporter"] + if not isinstance(ds4_binding, dict) or set(ds4_binding) != { + "id", "sha256", "install_trust_sha256"} or ds4_binding.get("id") != ( + expected_ds4_exporter_policy_id) or ds4_binding.get("sha256") != ( + ds4_digest) or re.fullmatch( + r"[0-9a-f]{64}", ds4_binding.get("install_trust_sha256", "")) is None: + raise TraceError("manifest ds4 exporter approval differs from external policy") if seen_run_ids is not None: if expected_run_id in seen_run_ids: raise TraceError("trace lane run ID was reused") @@ -1636,6 +1783,8 @@ def execution_authorization( required_approvals = {"prompt_builder"} if lane == CANDIDATE_LANE: required_approvals.add("candidate_exporter") + else: + required_approvals.add("ds4_exporter") if not isinstance(approvals, dict): raise TraceError("execution authorization approvals are invalid") _require_exact_keys(approvals, required_approvals, "execution authorization approvals") @@ -2027,8 +2176,10 @@ def seal_bundle( expected_challenge: str, expected_run_id: str, candidate_exporter_policies: dict[str, dict[str, Any]] = APPROVED_CANDIDATE_EXPORTERS, + ds4_exporter_policies: dict[str, dict[str, Any]] = APPROVED_DS4_EXPORTERS, prompt_builder_policies: dict[str, dict[str, Any]] = APPROVED_PROMPT_BUILDERS, expected_candidate_exporter_policy_id: str | None = None, + expected_ds4_exporter_policy_id: str | None = None, expected_prompt_builder_policy_id: str = "", expected_approval_policy_sha256: str = "", expected_verifier_revision: str = "", @@ -2058,8 +2209,10 @@ def seal_bundle( expected_run_id=expected_run_id, verification_unix=verification_time, candidate_exporter_policies=candidate_exporter_policies, + ds4_exporter_policies=ds4_exporter_policies, prompt_builder_policies=prompt_builder_policies, expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, expected_approval_policy_sha256=expected_approval_policy_sha256, expected_verifier_revision=expected_verifier_revision, @@ -2079,8 +2232,10 @@ def seal_bundle( expected_run_id=expected_run_id, verification_unix=verification_time, candidate_exporter_policies=candidate_exporter_policies, + ds4_exporter_policies=ds4_exporter_policies, prompt_builder_policies=prompt_builder_policies, expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, expected_approval_policy_sha256=expected_approval_policy_sha256, expected_verifier_revision=expected_verifier_revision, @@ -2202,8 +2357,10 @@ def verify_bundle_seal( expected_run_id=verifier.expected_run_id, verification_unix=verifier.verification_unix, candidate_exporter_policies=verifier.candidate_exporter_policies, + ds4_exporter_policies=verifier.ds4_exporter_policies, prompt_builder_policies=verifier.prompt_builder_policies, expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=verifier.expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, expected_verifier_revision=verifier.expected_verifier_revision, @@ -2826,6 +2983,7 @@ def __init__( expected_challenge: str | None = None, expected_run_id: str | None = None, expected_candidate_exporter_policy_id: str | None = None, + expected_ds4_exporter_policy_id: str | None = None, expected_prompt_builder_policy_id: str | None = None, verification_unix: int | None = None, seen_run_ids: set[str] | None = None): @@ -2840,19 +2998,23 @@ def __init__( "external signer, lane, challenge, run ID, and prompt builder approval are required") if expected_lane == CANDIDATE_LANE and expected_candidate_exporter_policy_id is None: raise TraceError("external candidate exporter approval is required") + if expected_lane == ORACLE_LANE and expected_ds4_exporter_policy_id is None: + raise TraceError("external ds4 exporter approval is required") verifier = TraceVerifier.production( signer_principal, expected_lane=expected_lane, expected_challenge=expected_challenge, expected_run_id=expected_run_id, expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, verification_unix=verification_unix, seen_run_ids=seen_run_ids, ) elif any(value is not None for value in ( signer_principal, expected_lane, expected_challenge, expected_run_id, - expected_candidate_exporter_policy_id, expected_prompt_builder_policy_id, + expected_candidate_exporter_policy_id, expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id, verification_unix, seen_run_ids)): raise TraceError("trace verifier cannot be combined with separate verification inputs") self.signer_principal = verifier.principal @@ -2914,7 +3076,10 @@ def _validate_manifest(self) -> None: "audits", "expected", } - top_level.add("candidate" if self.manifest["runtime"] == "llama.cpp" else "host") + if self.manifest["runtime"] == "llama.cpp": + top_level.add("candidate") + else: + top_level.update({"host", "oracle"}) _require_exact_keys(self.manifest, top_level, f"{self.manifest['runtime']} manifest") if not isinstance(self.manifest["revision"], str) or not self.manifest["revision"]: raise TraceError("manifest revision is invalid") @@ -2932,15 +3097,16 @@ def _validate_manifest(self) -> None: "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor", } if self.manifest["runtime"] == "llama.cpp" - else {"compiler", "target", "path", "sha256"} + else { + "compiler", "target", "path", "sha256", + "runtime_profile", "runtime_receipt_sha256", + "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor", + } ) _require_exact_keys(self.manifest["build"], build_keys, "manifest build") build_sha256 = self.manifest["build"].get("sha256", "") if not isinstance(build_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", build_sha256) is None: raise TraceError("manifest build SHA-256 is invalid") - if self.manifest["runtime"] == "ds4" and ( - APPROVED_EXPORTERS.get(build_sha256) != DS4_REVISION): - raise TraceError("ds4 exporter is not approved for the pinned ds4 revision") for key in build_keys - { "sha256", "number", "runtime_profile", "runtime_receipt_sha256", "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor"}: @@ -3084,6 +3250,43 @@ def _validate_manifest(self) -> None: if not isinstance(receipt_sha256, str) or receipt_sha256 != sha256_bytes( canonical_json(receipt).encode("ascii")): raise TraceError("manifest runtime receipt SHA-256 is invalid") + else: + ds4_policy, _ds4_policy_sha256 = ds4_exporter_approval( + self.verifier.expected_ds4_exporter_policy_id or "", + policies=self.verifier.ds4_exporter_policies, + ) + build_evidence = { + key: self.manifest["build"][key] + for key in ( + "revision", + "path", + "sha256", + "runtime_profile", + "runtime_receipt_sha256", + "runtime_libraries", + "runtime_libraries_post", + ) + if key in self.manifest["build"] + } + build_evidence["revision"] = self.manifest["revision"] + validate_runtime_build_evidence( + build_evidence, + ds4_policy, + label="ds4 exporter", + ) + module_monitor = self.manifest["build"]["runtime_module_monitor"] + if not isinstance(module_monitor, dict): + raise TraceError("ds4 manifest runtime module monitor is invalid") + _require_exact_keys( + module_monitor, + {"mechanism", "checked_after_trace", "project_additions"}, + "ds4 manifest runtime module monitor", + ) + if module_monitor["mechanism"] != "dyld-add-image" or ( + module_monitor["checked_after_trace"] is not True) or ( + module_monitor["project_additions"] != []): + raise TraceError("ds4 manifest runtime module monitor is invalid") + receipt = ds4_policy["runtime_receipt"] for section in ("model", "prompt"): if not isinstance(self.manifest[section], dict): raise TraceError(f"manifest {section} is invalid") @@ -3344,6 +3547,100 @@ def _validate_manifest(self) -> None: self.manifest["authorization"]["approvals"]["candidate_exporter"][ "install_trust_sha256"] != candidate_trust_sha256): raise TraceError("candidate install trust evidence is not bound to authorization") + else: + oracle = self.manifest.get("oracle") + if not isinstance(oracle, dict): + raise TraceError("ds4 oracle attestation is missing") + _require_exact_keys( + oracle, + { + "repository", + "revision", + "verifier_revision", + "executable_path", + "executable_sha256", + "runtime_profile", + "runtime_build_sha256", + "runtime_libraries_sha256", + "runtime_receipt_sha256", + "exporter_approval_id", + "exporter_approval_sha256", + "install_trust", + "install_trust_sha256", + }, + "ds4 oracle attestation", + ) + for key in ( + "revision", + "verifier_revision", + "executable_sha256", + "runtime_libraries_sha256", + "runtime_build_sha256", + "runtime_receipt_sha256", + "exporter_approval_sha256", + "install_trust_sha256"): + value = oracle.get(key, "") + if not isinstance(value, str) or re.fullmatch( + r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: + raise TraceError(f"ds4 oracle {key} is invalid") + if oracle["repository"] != DS4_REPOSITORY or oracle["revision"] != DS4_REVISION or ( + oracle["revision"] == oracle["verifier_revision"]): + raise TraceError("ds4 oracle producer/verifier identity is invalid") + if re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", oracle.get("exporter_approval_id", "")) is None: + raise TraceError("ds4 exporter approval ID is invalid") + if oracle["executable_path"] != self.manifest["build"]["path"] or ( + oracle["executable_sha256"] != self.manifest["build"]["sha256"]): + raise TraceError("ds4 oracle executable identity does not match the trace build") + if oracle["runtime_profile"] != self.manifest["build"]["runtime_profile"]: + raise TraceError("ds4 oracle runtime profile does not match the trace build") + ds4_policy, ds4_policy_sha256 = ds4_exporter_approval( + self.verifier.expected_ds4_exporter_policy_id or "", + policies=self.verifier.ds4_exporter_policies, + ) + build_evidence = { + "revision": self.manifest["revision"], + "path": self.manifest["build"]["path"], + "sha256": self.manifest["build"]["sha256"], + "runtime_profile": self.manifest["build"]["runtime_profile"], + "runtime_receipt_sha256": self.manifest["build"]["runtime_receipt_sha256"], + "runtime_libraries": self.manifest["build"]["runtime_libraries"], + "runtime_libraries_post": self.manifest["build"]["runtime_libraries_post"], + } + if oracle["runtime_build_sha256"] != runtime_build_evidence_sha256( + build_evidence, ds4_policy, label="ds4 exporter"): + raise TraceError("ds4 oracle runtime build SHA-256 does not match the trace build") + runtime_libraries_sha256 = sha256_bytes( + canonical_json({ + "pre": self.manifest["build"]["runtime_libraries"], + "post": self.manifest["build"]["runtime_libraries_post"], + }).encode("ascii")) + if oracle["runtime_libraries_sha256"] != runtime_libraries_sha256 or ( + oracle["runtime_receipt_sha256"] != self.manifest["build"]["runtime_receipt_sha256"]): + raise TraceError("ds4 oracle runtime evidence does not match the trace build") + if oracle["exporter_approval_id"] != self.verifier.expected_ds4_exporter_policy_id or ( + oracle["exporter_approval_sha256"] != ds4_policy_sha256): + raise TraceError("ds4 exporter approval differs from external policy") + policy_checks = { + "repository": oracle["repository"], + "revision": oracle["revision"], + "executable_path": oracle["executable_path"], + "executable_sha256": oracle["executable_sha256"], + "runtime_profile": oracle["runtime_profile"], + "runtime_receipt": receipt, + } + for key, value in policy_checks.items(): + if ds4_policy[key] != value: + raise TraceError(f"ds4 oracle {key} differs from external exporter approval") + if oracle["verifier_revision"] != self.verifier.expected_verifier_revision: + raise TraceError("ds4 oracle verifier revision differs from external approval") + oracle_trust = validate_install_trust_evidence( + oracle["install_trust"], ds4_policy) + oracle_trust_sha256 = install_trust_sha256(oracle_trust) + if oracle["install_trust_sha256"] != oracle_trust_sha256 or ( + self.manifest["authorization"]["approvals"]["ds4_exporter"][ + "install_trust_sha256"] != oracle_trust_sha256): + raise TraceError("ds4 exporter install trust evidence is not bound to authorization") expected_config = { "layer_count": 40, "vocab_size": 129280, @@ -3562,6 +3859,14 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "runner_script_sha256", "exporter_path", "exporter_sha256", + "exporter_approval_id", + "exporter_approval_sha256", + "exporter_install_trust_sha256", + "exporter_runtime_build_sha256", + "exporter_runtime_profile", + "exporter_runtime_receipt_sha256", + "producer_revision", + "verifier_revision", "checkout_path", "checkout_revision", "command_sha256", @@ -3574,6 +3879,14 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "runtime_kind": "apple-metal", "source": "python-subprocess", "exporter_sha256": self.manifest["build"]["sha256"], + "exporter_approval_id": self.manifest["oracle"]["exporter_approval_id"], + "exporter_approval_sha256": self.manifest["oracle"]["exporter_approval_sha256"], + "exporter_install_trust_sha256": self.manifest["oracle"]["install_trust_sha256"], + "exporter_runtime_build_sha256": self.manifest["oracle"]["runtime_build_sha256"], + "exporter_runtime_profile": self.manifest["oracle"]["runtime_profile"], + "exporter_runtime_receipt_sha256": self.manifest["oracle"]["runtime_receipt_sha256"], + "producer_revision": self.manifest["oracle"]["revision"], + "verifier_revision": self.manifest["oracle"]["verifier_revision"], "checkout_revision": DS4_REVISION, "checkout_path": self.manifest["paths"]["runtime_checkout"], "runner_executable": self.manifest["paths"]["runner_executable"], @@ -3592,9 +3905,21 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "runner_executable_sha256", "runner_script_sha256", "exporter_sha256", + "exporter_approval_sha256", + "exporter_install_trust_sha256", + "exporter_runtime_build_sha256", + "exporter_runtime_receipt_sha256", "command_sha256"): if re.fullmatch(r"[0-9a-f]{64}", data.get(key, "")) is None: raise TraceError(f"{phase} ds4 runner {key} is invalid") + if re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", data.get("exporter_approval_id", "")) is None: + raise TraceError(f"{phase} ds4 runner exporter approval ID is invalid") + for key in ("producer_revision", "verifier_revision"): + if re.fullmatch(r"[0-9a-f]{40}", data.get(key, "")) is None: + raise TraceError(f"{phase} ds4 runner {key} is invalid") + if not isinstance(data.get("exporter_runtime_profile"), dict): + raise TraceError(f"{phase} ds4 runner runtime profile is invalid") if kind == "watchdog": required = ( "format", @@ -4181,6 +4506,7 @@ def command_validate(args: argparse.Namespace) -> int: expected_challenge=args.execution_challenge, expected_run_id=args.run_id, expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=args.ds4_exporter_policy_id, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, @@ -4220,6 +4546,7 @@ def command_compare(args: argparse.Namespace) -> int: expected_challenge=challenge, expected_run_id=left_run_id, expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=getattr(args, "left_ds4_exporter_policy_id", None), expected_prompt_builder_policy_id=getattr(args, "prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, ) @@ -4231,6 +4558,7 @@ def command_compare(args: argparse.Namespace) -> int: expected_run_id=right_run_id, expected_candidate_exporter_policy_id=getattr( args, "right_candidate_exporter_policy_id", None), + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=getattr(args, "prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, ) @@ -4243,6 +4571,7 @@ def command_compare(args: argparse.Namespace) -> int: expected_challenge=challenge, expected_run_id=left_run_id, expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=args.left_ds4_exporter_policy_id, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, @@ -4257,6 +4586,7 @@ def command_compare(args: argparse.Namespace) -> int: expected_challenge=challenge, expected_run_id=right_run_id, expected_candidate_exporter_policy_id=args.right_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=args.prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, @@ -4345,6 +4675,7 @@ def command_compare_local(args: argparse.Namespace) -> int: expected_run_id=left_run_id, expected_candidate_exporter_policy_id=getattr( args, "left_candidate_exporter_policy_id", None), + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=getattr( args, "left_prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, @@ -4357,6 +4688,7 @@ def command_compare_local(args: argparse.Namespace) -> int: expected_run_id=right_run_id, expected_candidate_exporter_policy_id=getattr( args, "right_candidate_exporter_policy_id", None), + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=getattr( args, "right_prompt_builder_policy_id", None), seen_run_ids=seen_run_ids, @@ -4370,6 +4702,7 @@ def command_compare_local(args: argparse.Namespace) -> int: expected_challenge=challenge, expected_run_id=left_run_id, expected_candidate_exporter_policy_id=args.left_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=args.left_prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, @@ -4384,6 +4717,7 @@ def command_compare_local(args: argparse.Namespace) -> int: expected_challenge=challenge, expected_run_id=right_run_id, expected_candidate_exporter_policy_id=args.right_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, expected_prompt_builder_policy_id=args.right_prompt_builder_policy_id, approval_policy=approval_policy, verification_unix=None, @@ -4418,6 +4752,7 @@ def build_parser() -> argparse.ArgumentParser: validate_parser.add_argument("--execution-challenge", required=True) validate_parser.add_argument("--run-id", required=True) validate_parser.add_argument("--candidate-exporter-policy-id") + validate_parser.add_argument("--ds4-exporter-policy-id") validate_parser.add_argument("--prompt-builder-policy-id", required=True) add_executable_approval_arguments(validate_parser) validate_parser.set_defaults(func=command_validate) @@ -4429,6 +4764,7 @@ def build_parser() -> argparse.ArgumentParser: compare_parser.add_argument("--execution-challenge", required=True) compare_parser.add_argument("--left-run-id", required=True) compare_parser.add_argument("--right-run-id", required=True) + compare_parser.add_argument("--left-ds4-exporter-policy-id", required=True) compare_parser.add_argument("--right-candidate-exporter-policy-id", required=True) compare_parser.add_argument("--prompt-builder-policy-id", required=True) add_executable_approval_arguments(compare_parser) From 5a7452b6a99484f19f2bfbba8b85693035764943 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 13:39:36 -0700 Subject: [PATCH 38/56] trace : attest ds4 failures before returning Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 149 ++++++++++++++++++++++++++++ tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/run_ds4.py | 88 +++++++++++++--- 3 files changed, 222 insertions(+), 17 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 56ce8a7ce001..e415c2806d89 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2117,6 +2117,7 @@ def test_ds4_exporter_revalidates_each_invocation(self) -> None: exporter=exporter, exporter_identity=identity, exporter_policy=policy, + timeout_seconds=30, check=True, capture_output=True, text=True, @@ -2134,6 +2135,7 @@ def test_ds4_exporter_revalidates_each_invocation(self) -> None: exporter=exporter, exporter_identity=identity, exporter_policy=policy, + timeout_seconds=30, check=True, capture_output=True, text=True, @@ -2173,6 +2175,7 @@ def swap_after_precheck(*_args: object, **_kwargs: object) -> subprocess.Complet exporter=exporter, exporter_identity=identity, exporter_policy=policy, + timeout_seconds=30, check=True, capture_output=True, text=True, @@ -2211,6 +2214,7 @@ def fail_after_swap(*_args: object, **_kwargs: object) -> subprocess.CompletedPr exporter=exporter, exporter_identity=identity, exporter_policy=policy, + timeout_seconds=30, check=True, capture_output=True, text=True, @@ -2237,6 +2241,7 @@ def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: exporter=exporter, exporter_identity=identity, exporter_policy=policy, + timeout_seconds=30, ) execute.assert_not_called() finally: @@ -3128,6 +3133,150 @@ def test_metal_accelerator_query_rejects_duplicate_keys(self) -> None: ) self.assertEqual(execute.call_count, 2) + def test_ds4_accelerator_query_attests_after_launch_exceptions(self) -> None: + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + for primary_error in ( + OSError("device launch failed"), + subprocess.TimeoutExpired(["exporter"], 7), + ): + with self.subTest(error=type(primary_error).__name__), mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[primary_error, build_result], + ) as execute, self.assertRaisesRegex(type(primary_error), "device launch failed|timed out"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 2) + self.assertEqual( + execute.call_args_list[0].args[0], + ["/approved/exporter", "--dsv41-attest-device", "Metal0"], + ) + self.assertEqual( + execute.call_args_list[1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + self.assertEqual( + execute.call_args_list[0].kwargs["timeout_seconds"], + run_ds4.EXPORTER_ATTESTATION_TIMEOUT_SECONDS, + ) + self.assertEqual( + execute.call_args_list[1].kwargs["timeout_seconds"], + run_ds4.EXPORTER_ATTESTATION_TIMEOUT_SECONDS, + ) + + def test_ds4_accelerator_query_attests_after_nonzero_exit(self) -> None: + device_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 9, "", "device failed") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[device_result, build_result], + ) as execute, self.assertRaisesRegex(preflight.PreflightError, "device failed"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 2) + + def test_ds4_invocation_reports_primary_and_post_attestation_failures(self) -> None: + primary_failures = ( + subprocess.TimeoutExpired(["exporter"], 7), + run_ds4.subprocess.CompletedProcess(["exporter"], 9, "", "device failed"), + ) + for primary_failure in primary_failures: + secondary_error = OSError("post-build launch failed") + with self.subTest(primary=type(primary_failure).__name__), mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[primary_failure, secondary_error], + ) as execute, self.assertRaisesRegex( + preflight.PreflightError, + "primary failure \\[(TimeoutExpired|PreflightError):.*secondary post-invocation.*" + "OSError: post-build launch failed"): + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--dsv41-attest-device", "Metal0"], + operation="selected accelerator query", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=7, + check=False, + ) + self.assertEqual(execute.call_count, 2) + self.assertEqual( + execute.call_args_list[1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + + def test_ds4_main_trace_result_waits_for_post_attestation(self) -> None: + trace_result = run_ds4.subprocess.CompletedProcess(["exporter"], 11, "", "trace failed") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[trace_result, build_result], + ) as execute: + result = run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertEqual(result.returncode, 11) + self.assertEqual(execute.call_count, 2) + self.assertEqual( + execute.call_args_list[1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + self.assertEqual( + execute.call_args_list[0].kwargs["timeout_seconds"], + run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + ) + + def test_ds4_exporter_command_requires_bounded_timeout(self) -> None: + completed = run_ds4.subprocess.CompletedProcess(["exporter"], 0, "", "") + identity = object() + with mock.patch.object( + run_ds4, "run_approved_executable", return_value=(completed, identity)) as execute: + result = run_ds4.run_exporter_command( + ["/approved/exporter"], + exporter=Path("/approved/exporter"), + exporter_identity=identity, + exporter_policy=DS4_EXPORTER_POLICY, + timeout_seconds=17, + check=False, + ) + self.assertIs(result, completed) + self.assertEqual(execute.call_args.kwargs["timeout"], 17) + for timeout in (None, 0, -1, True): + kwargs = {} if timeout is None else {"timeout_seconds": timeout} + with self.subTest(timeout=timeout), self.assertRaisesRegex( + preflight.PreflightError, "timeout is invalid"): + run_ds4.run_exporter_command( + ["/approved/exporter"], + exporter=Path("/approved/exporter"), + exporter_identity=identity, + exporter_policy=DS4_EXPORTER_POLICY, + **kwargs, + ) + def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 2d49d4cf9f10..d7e670735127 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored; if the invocation and post-attestation both fail, the launcher reports both failures. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 3e268c02d75c..377194c5156d 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -60,6 +60,10 @@ verify_approved_runtime_file_identities, ) +EXPORTER_ATTESTATION_TIMEOUT_SECONDS = 60 +EXPORTER_TRACE_TIMEOUT_SECONDS = 24 * 60 * 60 + + def git_output(checkout: Path, *args: str) -> str: try: return subprocess.check_output( @@ -133,7 +137,10 @@ def run_exporter_command( exporter: Path, exporter_identity: ExecutableFileReceipt, exporter_policy: dict[str, Any], + timeout_seconds: int | None = None, **kwargs: Any) -> subprocess.CompletedProcess[Any]: + if type(timeout_seconds) is not int or timeout_seconds <= 0 or "timeout" in kwargs: + raise PreflightError("ds4 exporter timeout is invalid") result, executed_identity = run_approved_executable( command, path=exporter, @@ -141,6 +148,7 @@ def run_exporter_command( expected_path=exporter_policy["executable_path"], expected_sha256=exporter_policy["executable_sha256"], label="ds4 exporter", + timeout=timeout_seconds, **kwargs, ) if executed_identity != exporter_identity: @@ -158,6 +166,7 @@ def query_runtime_build_attestation( exporter=exporter, exporter_identity=exporter_identity, exporter_policy=exporter_policy, + timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, check=False, capture_output=True, text=True, @@ -171,6 +180,61 @@ def query_runtime_build_attestation( raise PreflightError(f"ds4 exporter build attestation is invalid: {error}") from error +def run_exporter_with_post_attestation( + command: list[str], + *, + operation: str, + exporter: Path, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any], + expected_runtime_build: dict[str, Any], + timeout_seconds: int, + **kwargs: Any) -> subprocess.CompletedProcess[Any]: + result = None + primary_error = None + try: + result = run_exporter_command( + command, + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + timeout_seconds=timeout_seconds, + **kwargs, + ) + except (OSError, subprocess.SubprocessError, TraceError, PreflightError) as error: + primary_error = error + nonzero_error = None + if result is not None and result.returncode != 0: + detail = result.stderr.strip() if isinstance(result.stderr, str) else "" + nonzero_error = PreflightError( + f"{operation} failed: {detail or f'exit {result.returncode}'}") + secondary_error = None + try: + post_runtime_build = query_runtime_build_attestation( + exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + ) + if post_runtime_build != expected_runtime_build: + raise PreflightError(f"ds4 exporter build identity changed during {operation}") + except (OSError, subprocess.SubprocessError, TraceError, PreflightError) as error: + secondary_error = error + reported_primary = primary_error or nonzero_error + if reported_primary is not None: + if secondary_error is not None: + raise PreflightError( + f"{operation} primary failure [{type(reported_primary).__name__}: {reported_primary}]; " + f"secondary post-invocation runtime-build attestation failure " + f"[{type(secondary_error).__name__}: {secondary_error}]") from reported_primary + if primary_error is not None: + raise primary_error + if secondary_error is not None: + raise secondary_error + if result is None: + raise PreflightError(f"{operation} did not return a result") + return result + + def query_accelerator_attestation( exporter: Path, device: str, @@ -178,11 +242,14 @@ def query_accelerator_attestation( exporter_identity: ExecutableFileReceipt, exporter_policy: dict[str, Any], expected_runtime_build: dict[str, Any]) -> dict[str, object]: - result = run_exporter_command( + result = run_exporter_with_post_attestation( [str(exporter), "--dsv41-attest-device", device], + operation="selected accelerator query", exporter=exporter, exporter_identity=exporter_identity, exporter_policy=exporter_policy, + expected_runtime_build=expected_runtime_build, + timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, check=False, capture_output=True, text=True, @@ -199,13 +266,6 @@ def query_accelerator_attestation( except (TraceError, PreflightError) as error: validation_error = PreflightError( f"selected accelerator query returned invalid attestation: {error}") - post_runtime_build = query_runtime_build_attestation( - exporter, - exporter_identity=exporter_identity, - exporter_policy=exporter_policy, - ) - if post_runtime_build != expected_runtime_build: - raise PreflightError("ds4 exporter build identity changed during accelerator query") if validation_error is not None: raise validation_error if attestation is None: @@ -593,11 +653,14 @@ def main() -> int: pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) pre_audit_digests = seal_audits(pre_audits) print("exec:", shlex.join(command), file=sys.stderr) - result = run_exporter_command( + result = run_exporter_with_post_attestation( command, + operation="ds4 trace execution", exporter=exporter, exporter_identity=exporter_identity, exporter_policy=exporter_policy, + expected_runtime_build=pre_runtime_build, + timeout_seconds=EXPORTER_TRACE_TIMEOUT_SECONDS, cwd=checkout, check=False, ) @@ -605,13 +668,6 @@ def main() -> int: exporter, exporter_identity, label="ds4 exporter") verify_approved_runtime_file_identities( runtime_identities, label="ds4 exporter") - post_trace_runtime_build = query_runtime_build_attestation( - exporter, - exporter_identity=exporter_identity, - exporter_policy=exporter_policy, - ) - if post_trace_runtime_build != pre_runtime_build: - raise PreflightError("ds4 exporter build identity changed during trace execution") if result.returncode != 0: return result.returncode verify_sealed_audits(pre_audits, pre_audit_digests) From 2eaec20d0be8e5ed05772eac71dac661a45e8771 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 14:17:36 -0700 Subject: [PATCH 39/56] trace : contain ds4 process failures Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 468 +++++++++++++++-- tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/run_ds4.py | 40 +- tools/deepseek-v41-trace/trace_format.py | 633 ++++++++++++++++++++--- 4 files changed, 1039 insertions(+), 104 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index e415c2806d89..ba5df31caa57 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -3,6 +3,7 @@ import contextlib import copy import importlib.util +import inspect import io import json import os @@ -1991,10 +1992,11 @@ def test_approved_executable_uses_linux_descriptor_path(self) -> None: }], }, } - completed = subprocess.CompletedProcess([str(executable)], 0, "", "") + completed = subprocess.CompletedProcess([str(executable)], 0, b"", b"") + contained = trace._ContainedRun(completed, None, [], None) with isolated_test_install_trust(), mock.patch.object( trace.sys, "platform", "linux"), mock.patch.object( - trace.subprocess, "run", return_value=completed) as execute: + trace, "_run_contained_process", return_value=contained) as execute: result, identity = trace.run_approved_executable( [str(executable), "--version"], path=executable, @@ -2006,12 +2008,56 @@ def test_approved_executable_uses_linux_descriptor_path(self) -> None: capture_output=True, text=True, ) - self.assertIs(result, completed) + self.assertEqual(result.stdout, "") self.assertEqual(identity.path, str(executable)) - kwargs = execute.call_args.kwargs - self.assertRegex(kwargs["executable"], r"^/proc/self/fd/[0-9]+$") + launch = execute.call_args.kwargs["launch"] + self.assertRegex(launch["executable"], r"^/proc/self/fd/[0-9]+$") self.assertEqual(execute.call_args.args[0][0], str(executable)) - self.assertEqual(len(kwargs["pass_fds"]), 2) + self.assertEqual(len(launch["pass_fds"]), 2) + + def test_approved_executable_postchecks_before_strict_decode(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"approved") + executable.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": {"components": []}, + } + events = [] + completed = subprocess.CompletedProcess([str(executable)], 0, b"\xff", b"") + contained = trace._ContainedRun(completed, None, [], None) + original_verify = trace.verify_approved_executable_identity + original_decode = trace._decode_subprocess_stream + + def verify(*args: object, **kwargs: object) -> None: + events.append("verify") + original_verify(*args, **kwargs) + + def decode(*args: object, **kwargs: object) -> bytes | str | None: + events.append("decode") + return original_decode(*args, **kwargs) + + with isolated_test_install_trust(), mock.patch.object( + trace, "_run_contained_process", return_value=contained), mock.patch.object( + trace, "verify_approved_executable_identity", side_effect=verify), mock.patch.object( + trace, "_decode_subprocess_stream", side_effect=decode), self.assertRaises(UnicodeDecodeError): + trace.run_approved_executable( + [str(executable)], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + check=False, + capture_output=True, + text=True, + ) + self.assertGreater(events.count("verify"), 1) + self.assertLess(max(index for index, event in enumerate(events) if event == "verify"), events.index("decode")) def test_approved_install_rejects_mutable_alias_and_hardlink_paths(self) -> None: for mutation, message in ( @@ -2088,7 +2134,7 @@ def test_writable_install_root_blocks_replace_restore_before_launch(self) -> Non } with isolated_test_install_trust(), mock.patch.object( trace.sys, "platform", "linux"), mock.patch.object( - trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace, "_run_contained_process") as execute, self.assertRaisesRegex( trace.TraceError, "path is mutable"): trace.run_approved_executable( [str(executable)], @@ -2120,15 +2166,14 @@ def test_ds4_exporter_revalidates_each_invocation(self) -> None: timeout_seconds=30, check=True, capture_output=True, - text=True, ) - self.assertEqual(first.stdout, "approved\n") + self.assertEqual(first.stdout, b"approved\n") exporter.parent.chmod(0o755) exporter.chmod(0o755) exporter.write_text("#!/bin/sh\nprintf 'replacement\\n'\n", encoding="ascii") exporter.chmod(0o555) exporter.parent.chmod(0o555) - with mock.patch.object(trace.subprocess, "run") as execute, self.assertRaisesRegex( + with mock.patch.object(trace, "_run_contained_process") as execute, self.assertRaisesRegex( run_ds4.TraceError, "SHA-256 differs|identity changed"): run_ds4.run_exporter_command( [str(exporter)], @@ -2138,7 +2183,6 @@ def test_ds4_exporter_revalidates_each_invocation(self) -> None: timeout_seconds=30, check=True, capture_output=True, - text=True, ) execute.assert_not_called() @@ -2160,15 +2204,19 @@ def test_ds4_exporter_detects_swap_after_precheck(self) -> None: label="ds4 exporter", ) - def swap_after_precheck(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + def swap_after_precheck(*_args: object, **_kwargs: object) -> trace._ContainedRun: exporter.parent.chmod(0o755) exporter.rename(backup) replacement.rename(exporter) exporter.parent.chmod(0o555) - return subprocess.CompletedProcess([str(exporter)], 0, "replacement\n", "") + result = subprocess.CompletedProcess([str(exporter)], 0, b"replacement\n", b"") + return trace._ContainedRun(result, None, [], None) with mock.patch.object( - trace.subprocess, "run", side_effect=swap_after_precheck), self.assertRaisesRegex( + sys.modules["trace_format"], + "_run_contained_process", + side_effect=swap_after_precheck, + ), self.assertRaisesRegex( run_ds4.TraceError, "descriptor identity changed|SHA-256 differs|identity changed"): run_ds4.run_exporter_command( [str(exporter)], @@ -2178,7 +2226,6 @@ def swap_after_precheck(*_args: object, **_kwargs: object) -> subprocess.Complet timeout_seconds=30, check=True, capture_output=True, - text=True, ) def test_ds4_exporter_postchecks_failed_invocation(self) -> None: @@ -2199,16 +2246,22 @@ def test_ds4_exporter_postchecks_failed_invocation(self) -> None: label="ds4 exporter", ) - def fail_after_swap(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + def fail_after_swap(*_args: object, **_kwargs: object) -> trace._ContainedRun: exporter.parent.chmod(0o755) exporter.rename(backup) replacement.rename(exporter) exporter.parent.chmod(0o555) - raise subprocess.CalledProcessError(7, [str(exporter)]) + error = subprocess.TimeoutExpired([str(exporter)], 7) + return trace._ContainedRun(None, error, [], None) with mock.patch.object( - trace.subprocess, "run", side_effect=fail_after_swap), self.assertRaisesRegex( - run_ds4.TraceError, "descriptor identity changed|SHA-256 differs|identity changed"): + sys.modules["trace_format"], + "_run_contained_process", + side_effect=fail_after_swap, + ), self.assertRaisesRegex( + run_ds4.TraceError, + "primary failure \\[TimeoutExpired:.*secondary integrity failures:.*" + "executable-path-root.*(SHA-256 differs|identity changed)"): run_ds4.run_exporter_command( [str(exporter)], exporter=exporter, @@ -2217,9 +2270,177 @@ def fail_after_swap(*_args: object, **_kwargs: object) -> subprocess.CompletedPr timeout_seconds=30, check=True, capture_output=True, - text=True, ) + def test_ds4_exporter_aggregates_all_postcheck_failures(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + runtime_path = ( + Path(policy["install_root"]) / "lib" / + policy["runtime_receipt"]["components"][0]["filename"]) + primary = subprocess.TimeoutExpired([str(exporter)], 7) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + + def fail_and_mutate(*_args: object, **_kwargs: object) -> trace._ContainedRun: + exporter.chmod(0o755) + exporter.write_text("#!/bin/sh\nexit 9\n", encoding="ascii") + runtime_path.chmod(0o755) + runtime_path.write_bytes(b"mutated runtime") + Path(policy["install_root"]).chmod(0o777) + cleanup = trace._IntegrityFailure( + "process-tree-quiescence", trace.TraceError("cleanup deadline expired")) + return trace._ContainedRun(None, primary, [cleanup], None) + + with mock.patch.object( + sys.modules["trace_format"], + "_run_contained_process", + side_effect=fail_and_mutate, + ), self.assertRaises(sys.modules["trace_format"].ExecutionIntegrityError) as raised: + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + check=False, + capture_output=True, + ) + error = raised.exception + self.assertIs(error.__cause__, primary) + self.assertIs(error.primary_error, primary) + components = {failure.component for failure in error.secondary_errors} + self.assertIn("process-tree-quiescence", components) + self.assertIn("executable-descriptor", components) + self.assertIn("executable-path-root", components) + self.assertTrue(any(item.startswith("runtime-descriptor:") for item in components)) + self.assertTrue(any(item.startswith("runtime-path-root:") for item in components)) + + @unittest.skipUnless(os.name == "posix", "POSIX process-group test") + def test_approved_executable_timeout_kills_descendant_before_return(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + marker = root / "descendant-survived" + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + "( sleep 1; printf survived > \"$1\" ) /dev/null 2>&1 &\n" + "sleep 30\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + with isolated_test_install_trust(), self.assertRaises(subprocess.TimeoutExpired): + trace.run_approved_executable( + [str(exporter), str(marker)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="approved executable", + timeout=0.1, + check=False, + capture_output=True, + ) + time.sleep(1.2) + self.assertFalse(marker.exists()) + + def test_posix_process_tree_cleanup_escalates_and_reaps(self) -> None: + process = mock.Mock() + process.communicate.return_value = (b"", b"") + containment = trace._ProcessContainment(process=process, process_group_id=77) + with mock.patch.object( + trace, "_posix_process_group_exists", return_value=True), mock.patch.object( + trace, + "_wait_for_process_tree_quiescence", + side_effect=[trace.TraceError("term deadline"), None], + ), mock.patch.object(trace.os, "killpg") as killpg: + failures = trace._terminate_process_tree(containment) + self.assertEqual(failures, []) + self.assertEqual( + killpg.call_args_list, + [mock.call(77, trace.signal.SIGTERM), mock.call(77, trace.signal.SIGKILL)], + ) + process.communicate.assert_called_once() + + def test_posix_process_tree_cleanup_timeout_is_integrity_failure(self) -> None: + process = mock.Mock() + process.communicate.side_effect = subprocess.TimeoutExpired(["exporter"], 5) + containment = trace._ProcessContainment(process=process, process_group_id=78) + with mock.patch.object( + trace, "_posix_process_group_exists", return_value=True), mock.patch.object( + trace, + "_wait_for_process_tree_quiescence", + side_effect=trace.TraceError("cleanup deadline"), + ), mock.patch.object(trace.os, "killpg") as killpg: + failures = trace._terminate_process_tree(containment) + self.assertEqual( + [failure.component for failure in failures], + ["direct-child-reap", "process-tree-quiescence"], + ) + self.assertEqual( + killpg.call_args_list, + [mock.call(78, trace.signal.SIGTERM), mock.call(78, trace.signal.SIGKILL)], + ) + + def test_windows_job_containment_is_suspended_before_assignment_and_resume(self) -> None: + start_source = inspect.getsource(trace._start_windows_job_process) + create_source = inspect.getsource(trace._create_windows_kill_job) + cleanup_source = inspect.getsource(trace._terminate_process_tree) + self.assertLess(start_source.index("subprocess.Popen"), start_source.index("AssignProcessToJobObject")) + self.assertLess(start_source.index("AssignProcessToJobObject"), start_source.index("ResumeThread")) + self.assertIn("WINDOWS_CREATE_SUSPENDED", start_source) + self.assertIn("WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE", create_source) + self.assertIn("SetInformationJobObject", create_source) + self.assertIn("TerminateJobObject", cleanup_source) + self.assertIn("QueryInformationJobObject", inspect.getsource(trace._windows_job_active_processes)) + self.assertIn("CloseHandle", inspect.getsource(trace._close_process_containment)) + sentinel = mock.Mock() + launch = {} + with mock.patch.object(trace.sys, "platform", "win32"), mock.patch.object( + trace, "_start_windows_job_process", return_value=sentinel) as start: + self.assertIs(trace._start_contained_process(["approved"], launch), sentinel) + start.assert_called_once_with(["approved"], launch) + + def test_windows_job_assignment_happens_before_resume(self) -> None: + events = [] + process = mock.Mock(pid=91) + process._handle = 92 + kernel32 = mock.Mock() + kernel32.AssignProcessToJobObject.side_effect = lambda *_args: events.append("assign") or 1 + kernel32.ResumeThread.side_effect = lambda *_args: events.append("resume") or 0 + kernel32.CloseHandle.side_effect = lambda *_args: events.append("close") or 1 + with mock.patch.object( + trace.subprocess, + "Popen", + side_effect=lambda *_args, **_kwargs: events.append("popen") or process, + ) as popen, mock.patch.object( + trace, + "_create_windows_kill_job", + side_effect=lambda: events.append("job") or 93, + ), mock.patch.object( + trace, "_windows_kernel32", return_value=kernel32), mock.patch.object( + trace, + "_open_windows_process_thread", + side_effect=lambda _pid: events.append("thread") or 94, + ): + containment = trace._start_windows_job_process(["approved"], {}) + self.assertEqual(events, ["popen", "job", "assign", "thread", "resume", "close"]) + self.assertEqual( + popen.call_args.kwargs["creationflags"] & trace.WINDOWS_CREATE_SUSPENDED, + trace.WINDOWS_CREATE_SUSPENDED, + ) + self.assertEqual(containment.job_handle, 93) + def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: with tempfile.TemporaryDirectory() as temp: policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) @@ -2234,7 +2455,7 @@ def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: ) Path(policy["install_root"]).chmod(0o777) try: - with mock.patch.object(trace.subprocess, "run") as execute, self.assertRaisesRegex( + with mock.patch.object(trace, "_run_contained_process") as execute, self.assertRaisesRegex( run_ds4.TraceError, "path is mutable|distinct"): run_ds4.run_exporter_command( [str(exporter)], @@ -2271,7 +2492,7 @@ def test_ds4_exporter_rejects_hardlinks_and_dependency_substitution(self) -> Non alias.symlink_to(exporter) exporter = alias with isolated_test_install_trust(), mock.patch.object( - trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace, "_run_contained_process") as execute, self.assertRaisesRegex( trace.TraceError, message): trace.run_approved_executable( [str(exporter)], @@ -2290,7 +2511,7 @@ def test_ds4_exporter_rejects_command_path_substitution(self) -> None: replacement.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") replacement.chmod(0o555) with isolated_test_install_trust(), mock.patch.object( - trace.subprocess, "run") as execute, self.assertRaisesRegex( + trace, "_run_contained_process") as execute, self.assertRaisesRegex( trace.TraceError, "command path differs"): trace.run_approved_executable( [str(replacement)], @@ -2305,7 +2526,7 @@ def test_ds4_exporter_rejects_command_path_substitution(self) -> None: def test_ds4_exporter_rejects_execution_owned_install(self) -> None: with tempfile.TemporaryDirectory() as temp: policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) - with mock.patch.object(trace.subprocess, "run") as execute, self.assertRaisesRegex( + with mock.patch.object(trace, "_run_contained_process") as execute, self.assertRaisesRegex( trace.TraceError, "owner must be distinct"): trace.run_approved_executable( [str(exporter)], @@ -3117,9 +3338,9 @@ def test_metal_accelerator_query_rejects_duplicate_keys(self) -> None: '"backend": "Metal"', '"backend": "Metal", "backend": "Metal"', ) - device_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, duplicate, "") + device_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, duplicate.encode("utf-8"), b"") build_result = run_ds4.subprocess.CompletedProcess( - ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") with mock.patch.object( run_ds4, "run_exporter_command", side_effect=[device_result, build_result]) as execute: @@ -3133,9 +3354,190 @@ def test_metal_accelerator_query_rejects_duplicate_keys(self) -> None: ) self.assertEqual(execute.call_count, 2) + def test_ds4_invalid_utf8_device_output_still_post_attests(self) -> None: + device_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[device_result, build_result], + ) as execute, self.assertRaisesRegex(preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual( + [call.args[0] for call in execute.call_args_list], + [ + ["/approved/exporter", "--dsv41-attest-device", "Metal0"], + ["/approved/exporter", "--dsv41-attest-build"], + ], + ) + + @unittest.skipUnless(os.name == "posix", "POSIX executable test") + def test_ds4_invalid_utf8_actual_process_still_runs_build_attestation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + marker = root / "build-attested" + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + "if test \"$1\" = --dsv41-attest-build; then\n" + f" printf build > '{marker}'\n" + " printf '{}'\n" + "else\n" + " printf '\\377'\n" + "fi\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + expected_build = fixture_runtime_build(policy) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + with mock.patch.object( + run_ds4, "validate_runtime_build_evidence", return_value=expected_build), self.assertRaisesRegex( + preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_accelerator_attestation( + exporter, + "Metal0", + exporter_identity=identity, + exporter_policy=policy, + expected_runtime_build=expected_build, + ) + self.assertEqual(marker.read_text(encoding="ascii"), "build") + + def test_ds4_invalid_utf8_build_attestation_is_nonrecursive(self) -> None: + invalid_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") + with mock.patch.object( + run_ds4, "run_exporter_command", return_value=invalid_result) as execute, self.assertRaisesRegex( + preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_runtime_build_attestation( + Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + ) + execute.assert_called_once() + + def test_ds4_main_unicode_error_still_post_attests(self) -> None: + primary = UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[primary, build_result], + ) as execute, self.assertRaises(UnicodeDecodeError) as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertIs(raised.exception, primary) + self.assertEqual(execute.call_count, 2) + + def test_ds4_postflight_invalid_utf8_still_post_attests(self) -> None: + valid_device = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, json.dumps(METAL_ACCELERATOR_ATTESTATION).encode("utf-8"), b"") + invalid_device = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[valid_device, build_result, invalid_device, build_result], + ) as execute: + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + with self.assertRaisesRegex(preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 4) + self.assertEqual( + execute.call_args_list[-1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + + def test_ds4_unicode_and_post_attestation_failures_are_both_retained(self) -> None: + primary = UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") + secondary = OSError("post-build launch failed") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[primary, secondary], + ) as execute, self.assertRaisesRegex( + preflight.PreflightError, + "primary failure \\[UnicodeDecodeError:.*secondary post-invocation.*" + "OSError: post-build launch failed") as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertIs(raised.exception.__cause__, primary) + self.assertEqual(execute.call_count, 2) + + def test_ds4_does_not_post_attest_without_process_tree_quiescence(self) -> None: + runtime_trace = sys.modules["trace_format"] + primary = subprocess.TimeoutExpired(["exporter"], 7) + failure = runtime_trace._IntegrityFailure( + "process-tree-quiescence", runtime_trace.TraceError("descendant survived")) + containment_error = runtime_trace.ExecutionIntegrityError( + "timeout and quiescence failure", + primary_error=primary, + secondary_errors=[failure], + ) + with mock.patch.object( + run_ds4, "run_exporter_command", side_effect=containment_error) as execute, self.assertRaises( + runtime_trace.ExecutionIntegrityError) as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertIs(raised.exception, containment_error) + execute.assert_called_once() + def test_ds4_accelerator_query_attests_after_launch_exceptions(self) -> None: build_result = run_ds4.subprocess.CompletedProcess( - ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") for primary_error in ( OSError("device launch failed"), subprocess.TimeoutExpired(["exporter"], 7), @@ -3172,9 +3574,9 @@ def test_ds4_accelerator_query_attests_after_launch_exceptions(self) -> None: def test_ds4_accelerator_query_attests_after_nonzero_exit(self) -> None: device_result = run_ds4.subprocess.CompletedProcess( - ["exporter"], 9, "", "device failed") + ["exporter"], 9, b"", b"device failed") build_result = run_ds4.subprocess.CompletedProcess( - ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") with mock.patch.object( run_ds4, "run_exporter_command", @@ -3192,7 +3594,7 @@ def test_ds4_accelerator_query_attests_after_nonzero_exit(self) -> None: def test_ds4_invocation_reports_primary_and_post_attestation_failures(self) -> None: primary_failures = ( subprocess.TimeoutExpired(["exporter"], 7), - run_ds4.subprocess.CompletedProcess(["exporter"], 9, "", "device failed"), + run_ds4.subprocess.CompletedProcess(["exporter"], 9, b"", b"device failed"), ) for primary_failure in primary_failures: secondary_error = OSError("post-build launch failed") @@ -3221,9 +3623,9 @@ def test_ds4_invocation_reports_primary_and_post_attestation_failures(self) -> N ) def test_ds4_main_trace_result_waits_for_post_attestation(self) -> None: - trace_result = run_ds4.subprocess.CompletedProcess(["exporter"], 11, "", "trace failed") + trace_result = run_ds4.subprocess.CompletedProcess(["exporter"], 11, b"", b"trace failed") build_result = run_ds4.subprocess.CompletedProcess( - ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD), "") + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") with mock.patch.object( run_ds4, "run_exporter_command", @@ -3251,7 +3653,7 @@ def test_ds4_main_trace_result_waits_for_post_attestation(self) -> None: ) def test_ds4_exporter_command_requires_bounded_timeout(self) -> None: - completed = run_ds4.subprocess.CompletedProcess(["exporter"], 0, "", "") + completed = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"", b"") identity = object() with mock.patch.object( run_ds4, "run_approved_executable", return_value=(completed, identity)) as execute: diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index d7e670735127..e7d0fdf3260b 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored; if the invocation and post-attestation both fail, the launcher reports both failures. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Approved processes run in a new POSIX session and process group; the Windows containment implementation creates the process suspended, assigns it to a kill-on-close Job Object, and resumes it only after assignment. A timeout or abnormal execution terminates and reaps the full contained tree before postchecks, and failure to prove quiescence blocks post-attestation. Captured output remains bytes until process-tree cleanup and all lower identity checks finish, then DS4 output is decoded as strict UTF-8 without replacement. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 377194c5156d..6e714ac0a245 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -28,6 +28,7 @@ CORPUS_SHA256, DS4_REPOSITORY, DS4_REVISION, + ExecutionIntegrityError, ExecutableFileReceipt, MODEL_SHA256, NO_EXTERNAL_STATE_STORAGE, @@ -139,7 +140,8 @@ def run_exporter_command( exporter_policy: dict[str, Any], timeout_seconds: int | None = None, **kwargs: Any) -> subprocess.CompletedProcess[Any]: - if type(timeout_seconds) is not int or timeout_seconds <= 0 or "timeout" in kwargs: + if type(timeout_seconds) is not int or timeout_seconds <= 0 or ( + {"timeout", "text", "encoding", "errors", "universal_newlines"} & kwargs.keys()): raise PreflightError("ds4 exporter timeout is invalid") result, executed_identity = run_approved_executable( command, @@ -156,6 +158,15 @@ def run_exporter_command( return result +def decode_exporter_output(value: bytes | None, *, label: str) -> str: + if not isinstance(value, bytes): + raise PreflightError(f"{label} bytes are missing") + try: + return value.decode("utf-8", "strict") + except UnicodeError as error: + raise PreflightError(f"{label} is not valid UTF-8: {error}") from error + + def query_runtime_build_attestation( exporter: Path, *, @@ -169,12 +180,14 @@ def query_runtime_build_attestation( timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, check=False, capture_output=True, - text=True, ) if result.returncode != 0: - raise PreflightError(f"ds4 exporter build attestation failed: {result.stderr.strip()}") + detail = decode_exporter_output( + result.stderr, label="ds4 exporter build attestation stderr").strip() + raise PreflightError(f"ds4 exporter build attestation failed: {detail}") try: - record = strict_json_loads(result.stdout) + record = strict_json_loads(decode_exporter_output( + result.stdout, label="ds4 exporter build attestation stdout")) return validate_runtime_build_evidence(record, exporter_policy, label="ds4 exporter") except TraceError as error: raise PreflightError(f"ds4 exporter build attestation is invalid: {error}") from error @@ -201,13 +214,15 @@ def run_exporter_with_post_attestation( timeout_seconds=timeout_seconds, **kwargs, ) - except (OSError, subprocess.SubprocessError, TraceError, PreflightError) as error: + except BaseException as error: primary_error = error + if isinstance(primary_error, ExecutionIntegrityError) and any( + failure.component.startswith(("process-tree-", "direct-child-", "containment-")) + for failure in primary_error.secondary_errors): + raise primary_error nonzero_error = None if result is not None and result.returncode != 0: - detail = result.stderr.strip() if isinstance(result.stderr, str) else "" - nonzero_error = PreflightError( - f"{operation} failed: {detail or f'exit {result.returncode}'}") + nonzero_error = PreflightError(f"{operation} failed: exit {result.returncode}") secondary_error = None try: post_runtime_build = query_runtime_build_attestation( @@ -217,7 +232,7 @@ def run_exporter_with_post_attestation( ) if post_runtime_build != expected_runtime_build: raise PreflightError(f"ds4 exporter build identity changed during {operation}") - except (OSError, subprocess.SubprocessError, TraceError, PreflightError) as error: + except BaseException as error: secondary_error = error reported_primary = primary_error or nonzero_error if reported_primary is not None: @@ -252,16 +267,17 @@ def query_accelerator_attestation( timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, check=False, capture_output=True, - text=True, ) validation_error = None attestation = None if result.returncode != 0: - detail = result.stderr.strip() or f"exit {result.returncode}" + detail = decode_exporter_output( + result.stderr, label="selected accelerator query stderr").strip() or f"exit {result.returncode}" validation_error = PreflightError(f"selected accelerator query failed: {detail}") else: try: - record = strict_json_loads(result.stdout) + record = strict_json_loads(decode_exporter_output( + result.stdout, label="selected accelerator query stdout")) attestation = validate_accelerator_attestation(record, expected_device=device) except (TraceError, PreflightError) as error: validation_error = PreflightError( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 11e7d7d23e49..90347243fc99 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import argparse +import ctypes import hashlib import json import math import os import re +import signal import stat import struct import subprocess @@ -134,6 +136,45 @@ class TraceError(RuntimeError): pass +PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS = 5 +PROCESS_TREE_TERM_GRACE_SECONDS = 1 +WINDOWS_CREATE_SUSPENDED = 0x00000004 +WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 + + +@dataclass(frozen=True) +class _IntegrityFailure: + component: str + error: BaseException + + +class ExecutionIntegrityError(TraceError): + def __init__( + self, + message: str, + *, + primary_error: BaseException | None, + secondary_errors: list[_IntegrityFailure]): + super().__init__(message) + self.primary_error = primary_error + self.secondary_errors = tuple(secondary_errors) + + +@dataclass +class _ProcessContainment: + process: subprocess.Popen[bytes] + process_group_id: int | None = None + job_handle: int | None = None + + +@dataclass +class _ContainedRun: + result: subprocess.CompletedProcess[bytes] | None + primary_error: BaseException | None + integrity_failures: list[_IntegrityFailure] + containment: _ProcessContainment | None + + @dataclass(frozen=True) class TraceVerifier: principal: str @@ -595,6 +636,407 @@ def verify_approved_runtime_file_identities( raise TraceError(f"{label} runtime component identity changed after approval") +def _windows_error(message: str) -> TraceError: + return TraceError(f"{message}: Windows error {ctypes.get_last_error()}") + + +def _windows_kernel32() -> Any: + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR] + kernel32.CreateJobObjectW.restype = wintypes.HANDLE + kernel32.SetInformationJobObject.argtypes = [ + wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD] + kernel32.SetInformationJobObject.restype = wintypes.BOOL + kernel32.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE] + kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + kernel32.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT] + kernel32.TerminateJobObject.restype = wintypes.BOOL + kernel32.QueryInformationJobObject.argtypes = [ + wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD, ctypes.c_void_p] + kernel32.QueryInformationJobObject.restype = wintypes.BOOL + kernel32.CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD] + kernel32.CreateToolhelp32Snapshot.restype = wintypes.HANDLE + kernel32.Thread32First.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + kernel32.Thread32First.restype = wintypes.BOOL + kernel32.Thread32Next.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + kernel32.Thread32Next.restype = wintypes.BOOL + kernel32.OpenThread.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel32.OpenThread.restype = wintypes.HANDLE + kernel32.ResumeThread.argtypes = [wintypes.HANDLE] + kernel32.ResumeThread.restype = wintypes.DWORD + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + return kernel32 + + +def _create_windows_kill_job() -> int: + from ctypes import wintypes + + class BasicLimitInformation(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_int64), + ("PerJobUserTimeLimit", ctypes.c_int64), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class IoCounters(ctypes.Structure): + _fields_ = [ + ("ReadOperationCount", ctypes.c_uint64), + ("WriteOperationCount", ctypes.c_uint64), + ("OtherOperationCount", ctypes.c_uint64), + ("ReadTransferCount", ctypes.c_uint64), + ("WriteTransferCount", ctypes.c_uint64), + ("OtherTransferCount", ctypes.c_uint64), + ] + + class ExtendedLimitInformation(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", BasicLimitInformation), + ("IoInfo", IoCounters), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + kernel32 = _windows_kernel32() + job = kernel32.CreateJobObjectW(None, None) + if not job: + raise _windows_error("cannot create process containment job") + limits = ExtendedLimitInformation() + limits.BasicLimitInformation.LimitFlags = WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if not kernel32.SetInformationJobObject( + job, 9, ctypes.byref(limits), ctypes.sizeof(limits)): + primary_error = _windows_error("cannot configure process containment job") + if not kernel32.CloseHandle(job): + close_error = _windows_error("cannot close unconfigured process containment job") + raise ExecutionIntegrityError( + f"Windows job configuration primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: windows-job-handle-close " + f"[{type(close_error).__name__}: {close_error}]", + primary_error=primary_error, + secondary_errors=[_IntegrityFailure("windows-job-handle-close", close_error)], + ) from primary_error + raise primary_error + return int(job) + + +def _open_windows_process_thread(process_id: int) -> int: + from ctypes import wintypes + + class ThreadEntry32(ctypes.Structure): + _fields_ = [ + ("dwSize", wintypes.DWORD), + ("cntUsage", wintypes.DWORD), + ("th32ThreadID", wintypes.DWORD), + ("th32OwnerProcessID", wintypes.DWORD), + ("tpBasePri", wintypes.LONG), + ("tpDeltaPri", wintypes.LONG), + ("dwFlags", wintypes.DWORD), + ] + + kernel32 = _windows_kernel32() + snapshot = kernel32.CreateToolhelp32Snapshot(0x00000004, 0) + if not snapshot or int(snapshot) == ctypes.c_void_p(-1).value: + raise _windows_error("cannot enumerate suspended process threads") + entry = ThreadEntry32() + entry.dwSize = ctypes.sizeof(entry) + thread = None + try: + found = kernel32.Thread32First(snapshot, ctypes.byref(entry)) + while found: + if entry.th32OwnerProcessID == process_id: + thread = kernel32.OpenThread(0x0002 | 0x00100000, False, entry.th32ThreadID) + if not thread: + raise _windows_error("cannot open suspended process thread") + break + found = kernel32.Thread32Next(snapshot, ctypes.byref(entry)) + except BaseException as primary_error: + if not kernel32.CloseHandle(snapshot): + close_error = _windows_error("cannot close process thread snapshot") + raise ExecutionIntegrityError( + f"Windows thread enumeration primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: windows-snapshot-handle-close " + f"[{type(close_error).__name__}: {close_error}]", + primary_error=primary_error, + secondary_errors=[_IntegrityFailure("windows-snapshot-handle-close", close_error)], + ) from primary_error + raise + if not kernel32.CloseHandle(snapshot): + close_error = _windows_error("cannot close process thread snapshot") + failures = [_IntegrityFailure("windows-snapshot-handle-close", close_error)] + if thread: + if not kernel32.CloseHandle(thread): + failures.append(_IntegrityFailure( + "windows-thread-handle-close", + _windows_error("cannot close suspended process thread"))) + if len(failures) > 1: + raise ExecutionIntegrityError( + f"Windows thread enumeration integrity failures: " + f"{_format_integrity_failures(failures)}", + primary_error=None, + secondary_errors=failures, + ) from close_error + raise close_error + if not thread: + raise TraceError("cannot find suspended process thread") + return int(thread) + + +def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + creationflags = int(launch.pop("creationflags", 0)) | WINDOWS_CREATE_SUSPENDED + process = subprocess.Popen(command, creationflags=creationflags, **launch) + job = None + thread = None + try: + job = _create_windows_kill_job() + kernel32 = _windows_kernel32() + process_handle = int(process._handle) + if not kernel32.AssignProcessToJobObject(job, process_handle): + raise _windows_error("cannot assign suspended process to containment job") + thread = _open_windows_process_thread(process.pid) + if kernel32.ResumeThread(thread) == 0xFFFFFFFF: + raise _windows_error("cannot resume contained process") + thread_closed = kernel32.CloseHandle(thread) + thread = None + if not thread_closed: + raise _windows_error("cannot close resumed process thread") + return _ProcessContainment(process=process, job_handle=job) + except BaseException as primary_error: + kernel32 = _windows_kernel32() + failures = [] + if thread is not None: + if not kernel32.CloseHandle(thread): + failures.append(_IntegrityFailure( + "windows-thread-handle-close", + _windows_error("cannot close suspended process thread"))) + if job is not None: + if not kernel32.TerminateJobObject(job, 1): + failures.append(_IntegrityFailure( + "windows-job-termination", + _windows_error("cannot terminate failed process containment job"))) + if not kernel32.CloseHandle(job): + failures.append(_IntegrityFailure( + "windows-job-handle-close", + _windows_error("cannot close failed process containment job"))) + else: + try: + process.kill() + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-termination", error)) + try: + process.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-reap", error)) + if failures: + raise ExecutionIntegrityError( + f"Windows containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: {_format_integrity_failures(failures)}", + primary_error=primary_error, + secondary_errors=failures, + ) from primary_error + raise + + +def _start_contained_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + if sys.platform == "win32": + return _start_windows_job_process(command, launch) + launch["start_new_session"] = True + process = subprocess.Popen(command, **launch) + return _ProcessContainment(process=process, process_group_id=process.pid) + + +def _posix_process_group_exists(process_group_id: int) -> bool: + try: + os.killpg(process_group_id, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +def _windows_job_active_processes(job_handle: int) -> int: + from ctypes import wintypes + + class BasicAccountingInformation(ctypes.Structure): + _fields_ = [ + ("TotalUserTime", ctypes.c_int64), + ("TotalKernelTime", ctypes.c_int64), + ("ThisPeriodTotalUserTime", ctypes.c_int64), + ("ThisPeriodTotalKernelTime", ctypes.c_int64), + ("TotalPageFaultCount", wintypes.DWORD), + ("TotalProcesses", wintypes.DWORD), + ("ActiveProcesses", wintypes.DWORD), + ("TotalTerminatedProcesses", wintypes.DWORD), + ] + + accounting = BasicAccountingInformation() + kernel32 = _windows_kernel32() + if not kernel32.QueryInformationJobObject( + job_handle, 1, ctypes.byref(accounting), ctypes.sizeof(accounting), None): + raise _windows_error("cannot query process containment job") + return int(accounting.ActiveProcesses) + + +def _process_tree_is_quiescent(containment: _ProcessContainment) -> bool: + if containment.job_handle is not None: + return _windows_job_active_processes(containment.job_handle) == 0 + if containment.process_group_id is None: + raise TraceError("process containment identity is missing") + return not _posix_process_group_exists(containment.process_group_id) + + +def _wait_for_process_tree_quiescence(containment: _ProcessContainment, deadline: float) -> None: + while not _process_tree_is_quiescent(containment): + if time.monotonic() >= deadline: + raise TraceError("process tree did not become quiescent before the cleanup deadline") + time.sleep(0.01) + + +def _terminate_process_tree(containment: _ProcessContainment) -> list[_IntegrityFailure]: + failures = [] + deadline = time.monotonic() + PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS + process = containment.process + try: + if containment.job_handle is not None: + if not _windows_kernel32().TerminateJobObject(containment.job_handle, 1): + raise _windows_error("cannot terminate process containment job") + elif containment.process_group_id is not None and _posix_process_group_exists(containment.process_group_id): + try: + os.killpg(containment.process_group_id, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + term_deadline = min(deadline, time.monotonic() + PROCESS_TREE_TERM_GRACE_SECONDS) + try: + _wait_for_process_tree_quiescence(containment, term_deadline) + except TraceError: + if _posix_process_group_exists(containment.process_group_id): + try: + os.killpg(containment.process_group_id, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + else: + raise TraceError("process containment identity is missing") + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-termination", error)) + try: + remaining = max(0.01, deadline - time.monotonic()) + process.communicate(timeout=remaining) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + try: + _wait_for_process_tree_quiescence(containment, deadline) + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-quiescence", error)) + return failures + + +def _run_contained_process( + command: list[str], + *, + label: str, + timeout: float | None, + input_data: bytes | None, + launch: dict[str, Any], +) -> _ContainedRun: + containment = None + try: + containment = _start_contained_process(command, launch) + except ExecutionIntegrityError as error: + return _ContainedRun( + None, + error.primary_error or error, + list(error.secondary_errors), + None, + ) + except BaseException as error: + return _ContainedRun(None, error, [], None) + try: + stdout, stderr = containment.process.communicate(input=input_data, timeout=timeout) + result = subprocess.CompletedProcess( + command, containment.process.returncode, stdout, stderr) + except BaseException as error: + failures = _terminate_process_tree(containment) + return _ContainedRun(None, error, failures, containment) + try: + if _process_tree_is_quiescent(containment): + return _ContainedRun(result, None, [], containment) + except BaseException as error: + failures = [_IntegrityFailure("process-tree-quiescence", error)] + else: + error = TraceError(f"{label} process tree remained active after direct child exit") + failures = [] + failures.extend(_terminate_process_tree(containment)) + return _ContainedRun(None, error, failures, containment) + + +def _close_process_containment(containment: _ProcessContainment | None) -> list[_IntegrityFailure]: + if containment is None or containment.job_handle is None: + return [] + try: + if not _windows_kernel32().CloseHandle(containment.job_handle): + raise _windows_error("cannot close process containment job") + containment.job_handle = None + return [] + except BaseException as error: + return [_IntegrityFailure("containment-handle-close", error)] + + +def _format_integrity_failures(failures: list[_IntegrityFailure]) -> str: + return "; ".join( + f"{failure.component} [{type(failure.error).__name__}: {failure.error}]" + for failure in failures) + + +def _raise_execution_integrity_failures( + *, + label: str, + primary_error: BaseException | None, + integrity_failures: list[_IntegrityFailure], +) -> None: + if primary_error is not None and integrity_failures: + raise ExecutionIntegrityError( + f"{label} primary failure [{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: {_format_integrity_failures(integrity_failures)}", + primary_error=primary_error, + secondary_errors=integrity_failures, + ) from primary_error + if primary_error is not None: + raise primary_error + if len(integrity_failures) == 1: + raise integrity_failures[0].error + if integrity_failures: + raise ExecutionIntegrityError( + f"{label} integrity failures: {_format_integrity_failures(integrity_failures)}", + primary_error=None, + secondary_errors=integrity_failures, + ) from integrity_failures[0].error + + +def _decode_subprocess_stream( + stream: bytes | None, + *, + text: bool, + encoding: str | None, + errors: str | None, +) -> bytes | str | None: + if stream is None or not text: + return stream + return stream.decode(encoding or "utf-8", errors or "strict") + + def run_approved_executable( command: list[str], *, @@ -605,12 +1047,38 @@ def run_approved_executable( label: str, **kwargs: Any, ) -> tuple[subprocess.CompletedProcess[Any], ExecutableFileReceipt]: - if sys.platform not in {"linux", "darwin"}: + if sys.platform not in {"linux", "darwin", "win32"}: raise TraceError(f"{label} immutable execution is unsupported on this platform") if not command or command[0] != str(path): raise TraceError(f"{label} command path differs from external approval") - if "executable" in kwargs or "pass_fds" in kwargs: + protected_controls = { + "creationflags", "executable", "pass_fds", "preexec_fn", "process_group", "start_new_session"} + if protected_controls & kwargs.keys(): raise TraceError(f"{label} execution parameters may not override immutable launch controls") + timeout = kwargs.pop("timeout", None) + check = bool(kwargs.pop("check", False)) + capture_output = bool(kwargs.pop("capture_output", False)) + text = bool(kwargs.pop("text", kwargs.pop("universal_newlines", False))) + encoding = kwargs.pop("encoding", None) + errors = kwargs.pop("errors", None) + input_value = kwargs.pop("input", None) + if encoding is not None or errors is not None: + text = True + if input_value is not None and "stdin" in kwargs: + raise TraceError(f"{label} execution input conflicts with stdin") + if capture_output and ("stdout" in kwargs or "stderr" in kwargs): + raise TraceError(f"{label} capture_output conflicts with stdout or stderr") + if capture_output: + kwargs["stdout"] = subprocess.PIPE + kwargs["stderr"] = subprocess.PIPE + if input_value is not None: + kwargs["stdin"] = subprocess.PIPE + if isinstance(input_value, str): + input_data = input_value.encode(encoding or "utf-8", errors or "strict") + elif input_value is None or isinstance(input_value, bytes): + input_data = input_value + else: + raise TraceError(f"{label} execution input is invalid") identity, descriptor = _approved_file_identity( path, install_root=Path(runtime_policy["install_root"]), @@ -621,6 +1089,10 @@ def run_approved_executable( executable=True, ) runtime_files: list[tuple[ExecutableFileReceipt, int]] = [] + containment = None + result = None + primary_error = None + integrity_failures: list[_IntegrityFailure] = [] try: for component in runtime_policy["runtime_receipt"]["components"]: runtime_path = Path(runtime_policy["install_root"]) / "lib" / component["filename"] @@ -641,70 +1113,115 @@ def run_approved_executable( label=f"{label} runtime component", ) retained_descriptors = (descriptor, *(item[1] for item in runtime_files)) - launch = { - "pass_fds": retained_descriptors, - **kwargs, - } + launch = dict(kwargs) + if sys.platform != "win32": + launch["pass_fds"] = retained_descriptors if sys.platform == "linux": launch["executable"] = f"/proc/self/fd/{descriptor}" - execution_error: OSError | subprocess.SubprocessError | None = None - result = None + contained = _run_contained_process( + command, + label=label, + timeout=timeout, + input_data=input_data, + launch=launch, + ) + containment = contained.containment + result = contained.result + primary_error = contained.primary_error + integrity_failures.extend(contained.integrity_failures) + if result is not None and check and result.returncode != 0 and primary_error is None: + primary_error = subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) try: - result = subprocess.run(command, **launch) - except (OSError, subprocess.SubprocessError) as error: - execution_error = error - descriptor_after = os.fstat(descriptor) - if ( - descriptor_after.st_dev, - descriptor_after.st_ino, - descriptor_after.st_size, - descriptor_after.st_mtime_ns, - descriptor_after.st_ctime_ns, - ) != ( - identity.device, - identity.inode, - identity.byte_count, - identity.modified_ns, - identity.changed_ns, - ): - raise TraceError(f"{label} descriptor identity changed during execution") - verify_approved_executable_identity(path, identity, label=label) - for runtime_identity, runtime_descriptor in runtime_files: - runtime_after = os.fstat(runtime_descriptor) + descriptor_after = os.fstat(descriptor) if ( - runtime_after.st_dev, - runtime_after.st_ino, - runtime_after.st_uid, - stat.S_IMODE(runtime_after.st_mode), - runtime_after.st_nlink, - runtime_after.st_size, - runtime_after.st_mtime_ns, - runtime_after.st_ctime_ns, + descriptor_after.st_dev, + descriptor_after.st_ino, + descriptor_after.st_size, + descriptor_after.st_mtime_ns, + descriptor_after.st_ctime_ns, ) != ( - runtime_identity.device, - runtime_identity.inode, - runtime_identity.owner_uid, - runtime_identity.mode, - runtime_identity.link_count, - runtime_identity.byte_count, - runtime_identity.modified_ns, - runtime_identity.changed_ns, + identity.device, + identity.inode, + identity.byte_count, + identity.modified_ns, + identity.changed_ns, ): - raise TraceError(f"{label} runtime component descriptor changed during execution") - verify_approved_executable_identity( - Path(runtime_identity.path), - runtime_identity, - label=f"{label} runtime component", - ) - if execution_error is not None: - raise execution_error - if result is None: - raise TraceError(f"{label} execution did not return a result") - return result, identity + raise TraceError(f"{label} descriptor identity changed during execution") + except BaseException as error: + integrity_failures.append(_IntegrityFailure("executable-descriptor", error)) + try: + verify_approved_executable_identity(path, identity, label=label) + except BaseException as error: + integrity_failures.append(_IntegrityFailure("executable-path-root", error)) + for runtime_identity, runtime_descriptor in runtime_files: + try: + runtime_after = os.fstat(runtime_descriptor) + if ( + runtime_after.st_dev, + runtime_after.st_ino, + runtime_after.st_uid, + stat.S_IMODE(runtime_after.st_mode), + runtime_after.st_nlink, + runtime_after.st_size, + runtime_after.st_mtime_ns, + runtime_after.st_ctime_ns, + ) != ( + runtime_identity.device, + runtime_identity.inode, + runtime_identity.owner_uid, + runtime_identity.mode, + runtime_identity.link_count, + runtime_identity.byte_count, + runtime_identity.modified_ns, + runtime_identity.changed_ns, + ): + raise TraceError(f"{label} runtime component descriptor changed during execution") + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"runtime-descriptor:{runtime_identity.path}", error)) + try: + verify_approved_executable_identity( + Path(runtime_identity.path), + runtime_identity, + label=f"{label} runtime component", + ) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"runtime-path-root:{runtime_identity.path}", error)) + except BaseException as error: + if primary_error is None: + primary_error = error + else: + integrity_failures.append(_IntegrityFailure("launcher-orchestration", error)) finally: for _runtime_identity, runtime_descriptor in runtime_files: - os.close(runtime_descriptor) - os.close(descriptor) + try: + os.close(runtime_descriptor) + except BaseException as error: + integrity_failures.append(_IntegrityFailure("runtime-descriptor-close", error)) + try: + os.close(descriptor) + except BaseException as error: + integrity_failures.append(_IntegrityFailure("executable-descriptor-close", error)) + integrity_failures.extend(_close_process_containment(containment)) + _raise_execution_integrity_failures( + label=label, + primary_error=primary_error, + integrity_failures=integrity_failures, + ) + if result is None: + raise TraceError(f"{label} execution did not return a result") + stdout = _decode_subprocess_stream( + result.stdout, text=text, encoding=encoding, errors=errors) + stderr = _decode_subprocess_stream( + result.stderr, text=text, encoding=encoding, errors=errors) + decoded_result = subprocess.CompletedProcess(result.args, result.returncode, stdout, stderr) + return decoded_result, identity def install_trust_evidence( From 3fb4a39a549fcce3ebe9a16e7fb5cbc8d2277257 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 14:56:13 -0700 Subject: [PATCH 40/56] trace : own approved process descendants Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 372 ++++++++++++++----- tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/run_ds4.py | 55 ++- tools/deepseek-v41-trace/trace_format.py | 448 +++++++++++++++++++---- 4 files changed, 716 insertions(+), 161 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index ba5df31caa57..2dd2ccdd4424 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -681,7 +681,7 @@ def fixture_runtime_build(policy: dict[str, object]) -> dict[str, object]: @contextlib.contextmanager -def isolated_test_install_trust(): +def isolated_test_install_trust(*, process_containment: bool = True): modules = (trace, sys.modules["trace_format"]) with contextlib.ExitStack() as stack: for module in modules: @@ -694,6 +694,8 @@ def isolated_test_install_trust(): module, "_path_is_writable_by_execution_identity", return_value=False)) stack.enter_context(mock.patch.object( module, "_has_access_control_entries", return_value=False)) + if process_containment and sys.platform == "darwin": + stack.enter_context(module._test_only_process_group_containment()) yield @@ -1993,7 +1995,7 @@ def test_approved_executable_uses_linux_descriptor_path(self) -> None: }, } completed = subprocess.CompletedProcess([str(executable)], 0, b"", b"") - contained = trace._ContainedRun(completed, None, [], None) + contained = trace._ContainedRun(completed, None, [], None, True, True) with isolated_test_install_trust(), mock.patch.object( trace.sys, "platform", "linux"), mock.patch.object( trace, "_run_contained_process", return_value=contained) as execute: @@ -2029,7 +2031,7 @@ def test_approved_executable_postchecks_before_strict_decode(self) -> None: } events = [] completed = subprocess.CompletedProcess([str(executable)], 0, b"\xff", b"") - contained = trace._ContainedRun(completed, None, [], None) + contained = trace._ContainedRun(completed, None, [], None, True, True) original_verify = trace.verify_approved_executable_identity original_decode = trace._decode_subprocess_stream @@ -2044,7 +2046,8 @@ def decode(*args: object, **kwargs: object) -> bytes | str | None: with isolated_test_install_trust(), mock.patch.object( trace, "_run_contained_process", return_value=contained), mock.patch.object( trace, "verify_approved_executable_identity", side_effect=verify), mock.patch.object( - trace, "_decode_subprocess_stream", side_effect=decode), self.assertRaises(UnicodeDecodeError): + trace, "_decode_subprocess_stream", side_effect=decode), self.assertRaises( + trace.ExecutionIntegrityError) as raised: trace.run_approved_executable( [str(executable)], path=executable, @@ -2056,6 +2059,8 @@ def decode(*args: object, **kwargs: object) -> bytes | str | None: capture_output=True, text=True, ) + self.assertIsInstance(raised.exception.__cause__, UnicodeDecodeError) + self.assertTrue(raised.exception.quiescence_proven) self.assertGreater(events.count("verify"), 1) self.assertLess(max(index for index, event in enumerate(events) if event == "verify"), events.index("decode")) @@ -2210,7 +2215,7 @@ def swap_after_precheck(*_args: object, **_kwargs: object) -> trace._ContainedRu replacement.rename(exporter) exporter.parent.chmod(0o555) result = subprocess.CompletedProcess([str(exporter)], 0, b"replacement\n", b"") - return trace._ContainedRun(result, None, [], None) + return trace._ContainedRun(result, None, [], None, True, True) with mock.patch.object( sys.modules["trace_format"], @@ -2252,7 +2257,7 @@ def fail_after_swap(*_args: object, **_kwargs: object) -> trace._ContainedRun: replacement.rename(exporter) exporter.parent.chmod(0o555) error = subprocess.TimeoutExpired([str(exporter)], 7) - return trace._ContainedRun(None, error, [], None) + return trace._ContainedRun(None, error, [], None, True, True) with mock.patch.object( sys.modules["trace_format"], @@ -2298,7 +2303,7 @@ def fail_and_mutate(*_args: object, **_kwargs: object) -> trace._ContainedRun: Path(policy["install_root"]).chmod(0o777) cleanup = trace._IntegrityFailure( "process-tree-quiescence", trace.TraceError("cleanup deadline expired")) - return trace._ContainedRun(None, primary, [cleanup], None) + return trace._ContainedRun(None, primary, [cleanup], None, False, True) with mock.patch.object( sys.modules["trace_format"], @@ -2324,22 +2329,40 @@ def fail_and_mutate(*_args: object, **_kwargs: object) -> trace._ContainedRun: self.assertTrue(any(item.startswith("runtime-descriptor:") for item in components)) self.assertTrue(any(item.startswith("runtime-path-root:") for item in components)) - @unittest.skipUnless(os.name == "posix", "POSIX process-group test") - def test_approved_executable_timeout_kills_descendant_before_return(self) -> None: + @unittest.skipUnless(os.name == "posix", "POSIX containment test") + def test_approved_executable_timeout_contains_setsid_descendant(self) -> None: with tempfile.TemporaryDirectory() as temp: root = Path(temp).resolve() policy, exporter = materialize_ds4_exporter_policy(root) marker = root / "descendant-survived" exporter.chmod(0o755) - exporter.write_text( - "#!/bin/sh\n" - "( sleep 1; printf survived > \"$1\" ) /dev/null 2>&1 &\n" - "sleep 30\n", - encoding="ascii", - ) + if sys.platform == "linux": + exporter.write_text( + f"#!{sys.executable}\n" + "import os\n" + "import signal\n" + "import sys\n" + "import time\n" + "if os.fork() == 0:\n" + " os.setsid()\n" + " signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + " time.sleep(1)\n" + " open(sys.argv[1], 'w', encoding='ascii').write('survived')\n" + " os._exit(0)\n" + "time.sleep(30)\n", + encoding="ascii", + ) + else: + exporter.write_text( + "#!/bin/sh\n" + "( sleep 1; printf survived > \"$1\" ) /dev/null 2>&1 &\n" + "sleep 30\n", + encoding="ascii", + ) exporter.chmod(0o555) policy["executable_sha256"] = trace.sha256_file(exporter) - with isolated_test_install_trust(), self.assertRaises(subprocess.TimeoutExpired): + with isolated_test_install_trust(), self.assertRaises( + trace.ExecutionIntegrityError) as raised: trace.run_approved_executable( [str(exporter), str(marker)], path=exporter, @@ -2351,46 +2374,101 @@ def test_approved_executable_timeout_kills_descendant_before_return(self) -> Non check=False, capture_output=True, ) + self.assertIsInstance(raised.exception.__cause__, subprocess.TimeoutExpired) time.sleep(1.2) self.assertFalse(marker.exists()) - def test_posix_process_tree_cleanup_escalates_and_reaps(self) -> None: - process = mock.Mock() - process.communicate.return_value = (b"", b"") - containment = trace._ProcessContainment(process=process, process_group_id=77) + def test_linux_subreaper_signals_stable_pidfds_not_numeric_ids(self) -> None: + process = mock.Mock(pid=77) + containment = trace._ProcessContainment( + process=process, linux_root_pidfd=90, linux_lock_held=True) with mock.patch.object( - trace, "_posix_process_group_exists", return_value=True), mock.patch.object( - trace, - "_wait_for_process_tree_quiescence", - side_effect=[trace.TraceError("term deadline"), None], - ), mock.patch.object(trace.os, "killpg") as killpg: - failures = trace._terminate_process_tree(containment) + trace, "_linux_direct_children", return_value={77, 78}), mock.patch.object( + trace, "_linux_open_pidfd", return_value=91) as open_pidfd, mock.patch.object( + trace, "_linux_signal_pidfd") as signal_pidfd, mock.patch.object( + trace.os, "close") as close, mock.patch.object(trace.os, "kill") as numeric_kill: + failures = trace._linux_signal_owned_children(containment, trace.signal.SIGKILL) self.assertEqual(failures, []) + open_pidfd.assert_called_once_with(78) self.assertEqual( - killpg.call_args_list, - [mock.call(77, trace.signal.SIGTERM), mock.call(77, trace.signal.SIGKILL)], - ) - process.communicate.assert_called_once() - - def test_posix_process_tree_cleanup_timeout_is_integrity_failure(self) -> None: - process = mock.Mock() - process.communicate.side_effect = subprocess.TimeoutExpired(["exporter"], 5) - containment = trace._ProcessContainment(process=process, process_group_id=78) - with mock.patch.object( - trace, "_posix_process_group_exists", return_value=True), mock.patch.object( - trace, - "_wait_for_process_tree_quiescence", - side_effect=trace.TraceError("cleanup deadline"), - ), mock.patch.object(trace.os, "killpg") as killpg: - failures = trace._terminate_process_tree(containment) - self.assertEqual( - [failure.component for failure in failures], - ["direct-child-reap", "process-tree-quiescence"], - ) - self.assertEqual( - killpg.call_args_list, - [mock.call(78, trace.signal.SIGTERM), mock.call(78, trace.signal.SIGKILL)], + signal_pidfd.call_args_list, + [mock.call(90, trace.signal.SIGKILL), mock.call(91, trace.signal.SIGKILL)], ) + close.assert_called_once_with(91) + numeric_kill.assert_not_called() + + def test_linux_subreaper_rejects_unrelated_child_ownership(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_linux_enable_child_subreaper"), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_linux_direct_children", return_value={42}), mock.patch.object( + trace.subprocess, "Popen") as popen, self.assertRaisesRegex( + trace.TraceError, "owns unrelated children"): + trace._start_linux_subreaper_process(["approved"], {}) + popen.assert_not_called() + lock.release.assert_called_once() + + def test_linux_subreaper_boundary_precedes_target_execution(self) -> None: + source = inspect.getsource(trace._start_linux_subreaper_process) + self.assertLess(source.index("_linux_enable_child_subreaper"), source.index("subprocess.Popen")) + self.assertLess(source.index("_linux_task_ids"), source.index("subprocess.Popen")) + self.assertLess(source.index("_linux_direct_children"), source.index("subprocess.Popen")) + self.assertIn("_linux_open_pidfd", source) + self.assertNotIn("killpg", source) + self.assertNotIn("os.kill(", inspect.getsource(trace._linux_signal_owned_children)) + + def test_linux_subreaper_without_pidfd_support_fails_before_launch(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_linux_enable_child_subreaper"), mock.patch.object( + trace, "_linux_require_pidfd_support", side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( + trace.subprocess, "Popen") as popen, self.assertRaisesRegex( + trace.TraceError, "pidfd unavailable"): + trace._start_linux_subreaper_process(["approved"], {}) + popen.assert_not_called() + lock.release.assert_called_once() + + def test_unproven_posix_containment_fails_closed_before_setsid_escape(self) -> None: + if sys.platform == "linux": + self.skipTest("Linux uses subreaper and pidfd containment") + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + marker = root / "escaped" + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + f"printf started > '{marker}'\n" + "sleep 30\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + with isolated_test_install_trust(process_containment=False): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + with self.assertRaisesRegex( + run_ds4.TraceError, "proven process containment is unavailable"): + run_ds4.run_exporter_command( + [str(exporter), str(marker)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=1, + check=False, + capture_output=True, + ) + self.assertFalse(marker.exists()) def test_windows_job_containment_is_suspended_before_assignment_and_resume(self) -> None: start_source = inspect.getsource(trace._start_windows_job_process) @@ -2440,6 +2518,30 @@ def test_windows_job_assignment_happens_before_resume(self) -> None: trace.WINDOWS_CREATE_SUSPENDED, ) self.assertEqual(containment.job_handle, 93) + self.assertTrue(containment.windows_job_assigned) + self.assertTrue(containment.windows_process_resumed) + + def test_windows_job_assignment_failure_kills_and_reaps_exact_child(self) -> None: + process = mock.Mock(pid=91) + process._handle = 92 + kernel32 = mock.Mock() + kernel32.AssignProcessToJobObject.return_value = 0 + kernel32.CloseHandle.return_value = 1 + with mock.patch.object( + trace.subprocess, "Popen", return_value=process), mock.patch.object( + trace, "_create_windows_kill_job", return_value=93), mock.patch.object( + trace, "_windows_kernel32", return_value=kernel32), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_windows_job_process(["approved"], {}) + self.assertFalse(raised.exception.quiescence_proven) + process.kill.assert_called_once() + process.wait.assert_called_once_with(timeout=trace.PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + kernel32.TerminateJobObject.assert_not_called() + self.assertEqual( + kernel32.CloseHandle.call_args_list, + [mock.call(92), mock.call(93)], + ) + self.assertIsNone(process._handle) def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -3419,6 +3521,53 @@ def test_ds4_invalid_utf8_actual_process_still_runs_build_attestation(self) -> N ) self.assertEqual(marker.read_text(encoding="ascii"), "build") + @unittest.skipUnless(os.name == "posix", "POSIX executable test") + def test_ds4_actual_invalid_utf8_and_post_build_failure_are_both_retained(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + "if test \"$1\" = --dsv41-attest-build; then\n" + " printf 'post-build failed' >&2\n" + " exit 9\n" + "fi\n" + "printf '\\377'\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + expected_build = fixture_runtime_build(policy) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + with self.assertRaisesRegex( + preflight.PreflightError, + "primary failure.*not valid UTF-8.*secondary post-invocation.*post-build failed", + ) as raised: + run_ds4.query_accelerator_attestation( + exporter, + "Metal0", + exporter_identity=identity, + exporter_policy=policy, + expected_runtime_build=expected_build, + ) + primary = raised.exception.__cause__ + self.assertIsInstance(primary, preflight.PreflightError) + self.assertIsInstance(primary.__cause__, UnicodeDecodeError) + self.assertIs(raised.exception.primary_error, primary) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["post-invocation-runtime-build-attestation"], + ) + def test_ds4_invalid_utf8_build_attestation_is_nonrecursive(self) -> None: invalid_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") with mock.patch.object( @@ -3432,14 +3581,21 @@ def test_ds4_invalid_utf8_build_attestation_is_nonrecursive(self) -> None: execute.assert_called_once() def test_ds4_main_unicode_error_still_post_attests(self) -> None: + runtime_trace = sys.modules["trace_format"] primary = UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") + contained_error = runtime_trace.ExecutionIntegrityError( + "decode failed after contained execution", + primary_error=primary, + secondary_errors=[], + quiescence_proven=True, + ) build_result = run_ds4.subprocess.CompletedProcess( ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") with mock.patch.object( run_ds4, "run_exporter_command", - side_effect=[primary, build_result], - ) as execute, self.assertRaises(UnicodeDecodeError) as raised: + side_effect=[contained_error, build_result], + ) as execute, self.assertRaises(runtime_trace.ExecutionIntegrityError) as raised: run_ds4.run_exporter_with_post_attestation( ["/approved/exporter", "--model", "/model.gguf"], operation="ds4 trace execution", @@ -3450,7 +3606,7 @@ def test_ds4_main_unicode_error_still_post_attests(self) -> None: timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, check=False, ) - self.assertIs(raised.exception, primary) + self.assertIs(raised.exception, contained_error) self.assertEqual(execute.call_count, 2) def test_ds4_postflight_invalid_utf8_still_post_attests(self) -> None: @@ -3486,15 +3642,16 @@ def test_ds4_postflight_invalid_utf8_still_post_attests(self) -> None: ) def test_ds4_unicode_and_post_attestation_failures_are_both_retained(self) -> None: - primary = UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") secondary = OSError("post-build launch failed") + invalid_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, b"\xff", b"") with mock.patch.object( run_ds4, "run_exporter_command", - side_effect=[primary, secondary], + side_effect=[invalid_result, secondary], ) as execute, self.assertRaisesRegex( preflight.PreflightError, - "primary failure \\[UnicodeDecodeError:.*secondary post-invocation.*" + "primary failure \\[PreflightError:.*not valid UTF-8.*secondary post-invocation.*" "OSError: post-build launch failed") as raised: run_ds4.run_exporter_with_post_attestation( ["/approved/exporter", "--model", "/model.gguf"], @@ -3505,48 +3662,72 @@ def test_ds4_unicode_and_post_attestation_failures_are_both_retained(self) -> No expected_runtime_build=DS4_RUNTIME_BUILD, timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, check=False, + capture_output=True, + decode_stdout_label="ds4 trace stdout", + decode_stderr_label="ds4 trace stderr", ) - self.assertIs(raised.exception.__cause__, primary) + self.assertIsInstance(raised.exception.__cause__, preflight.PreflightError) + self.assertIsInstance(raised.exception.__cause__.__cause__, UnicodeDecodeError) + self.assertIs(raised.exception.primary_error, raised.exception.__cause__) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["post-invocation-runtime-build-attestation"], + ) self.assertEqual(execute.call_count, 2) - def test_ds4_does_not_post_attest_without_process_tree_quiescence(self) -> None: + def test_ds4_explicit_quiescence_false_always_blocks_post_attestation(self) -> None: runtime_trace = sys.modules["trace_format"] - primary = subprocess.TimeoutExpired(["exporter"], 7) - failure = runtime_trace._IntegrityFailure( - "process-tree-quiescence", runtime_trace.TraceError("descendant survived")) - containment_error = runtime_trace.ExecutionIntegrityError( - "timeout and quiescence failure", - primary_error=primary, - secondary_errors=[failure], - ) - with mock.patch.object( - run_ds4, "run_exporter_command", side_effect=containment_error) as execute, self.assertRaises( - runtime_trace.ExecutionIntegrityError) as raised: - run_ds4.run_exporter_with_post_attestation( - ["/approved/exporter", "--model", "/model.gguf"], - operation="ds4 trace execution", - exporter=Path("/approved/exporter"), - exporter_identity=object(), - exporter_policy=DS4_EXPORTER_POLICY, - expected_runtime_build=DS4_RUNTIME_BUILD, - timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, - check=False, - ) - self.assertIs(raised.exception, containment_error) - execute.assert_called_once() + for component in ( + "process-tree-quiescence", + "windows-process-reap", + "windows-process-termination", + "windows-job-assignment", + "linux-child-ownership"): + with self.subTest(component=component): + primary = subprocess.TimeoutExpired(["exporter"], 7) + failure = runtime_trace._IntegrityFailure( + component, runtime_trace.TraceError("quiescence not proven")) + containment_error = runtime_trace.ExecutionIntegrityError( + "execution did not prove quiescence", + primary_error=primary, + secondary_errors=[failure], + quiescence_proven=False, + ) + with mock.patch.object( + run_ds4, "run_exporter_command", side_effect=containment_error) as execute, self.assertRaises( + runtime_trace.ExecutionIntegrityError) as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertIs(raised.exception, containment_error) + execute.assert_called_once() def test_ds4_accelerator_query_attests_after_launch_exceptions(self) -> None: + runtime_trace = sys.modules["trace_format"] build_result = run_ds4.subprocess.CompletedProcess( ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") - for primary_error in ( + for primary in ( OSError("device launch failed"), subprocess.TimeoutExpired(["exporter"], 7), ): - with self.subTest(error=type(primary_error).__name__), mock.patch.object( + primary_error = runtime_trace.ExecutionIntegrityError( + "contained invocation failed", + primary_error=primary, + secondary_errors=[], + quiescence_proven=True, + ) + with self.subTest(error=type(primary).__name__), mock.patch.object( run_ds4, "run_exporter_command", side_effect=[primary_error, build_result], - ) as execute, self.assertRaisesRegex(type(primary_error), "device launch failed|timed out"): + ) as execute, self.assertRaises(runtime_trace.ExecutionIntegrityError): run_ds4.query_accelerator_attestation( Path("/approved/exporter"), "Metal0", @@ -3572,6 +3753,21 @@ def test_ds4_accelerator_query_attests_after_launch_exceptions(self) -> None: run_ds4.EXPORTER_ATTESTATION_TIMEOUT_SECONDS, ) + def test_ds4_pre_spawn_failure_does_not_launch_post_attestation(self) -> None: + primary = OSError("process creation failed") + with mock.patch.object( + run_ds4, "run_exporter_command", side_effect=primary) as execute, self.assertRaises( + OSError) as raised: + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertIs(raised.exception, primary) + execute.assert_called_once() + def test_ds4_accelerator_query_attests_after_nonzero_exit(self) -> None: device_result = run_ds4.subprocess.CompletedProcess( ["exporter"], 9, b"", b"device failed") @@ -3592,8 +3788,14 @@ def test_ds4_accelerator_query_attests_after_nonzero_exit(self) -> None: self.assertEqual(execute.call_count, 2) def test_ds4_invocation_reports_primary_and_post_attestation_failures(self) -> None: + runtime_trace = sys.modules["trace_format"] primary_failures = ( - subprocess.TimeoutExpired(["exporter"], 7), + runtime_trace.ExecutionIntegrityError( + "contained invocation timed out", + primary_error=subprocess.TimeoutExpired(["exporter"], 7), + secondary_errors=[], + quiescence_proven=True, + ), run_ds4.subprocess.CompletedProcess(["exporter"], 9, b"", b"device failed"), ) for primary_failure in primary_failures: @@ -3604,7 +3806,7 @@ def test_ds4_invocation_reports_primary_and_post_attestation_failures(self) -> N side_effect=[primary_failure, secondary_error], ) as execute, self.assertRaisesRegex( preflight.PreflightError, - "primary failure \\[(TimeoutExpired|PreflightError):.*secondary post-invocation.*" + "primary failure \\[(ExecutionIntegrityError|PreflightError):.*secondary post-invocation.*" "OSError: post-build launch failed"): run_ds4.run_exporter_with_post_attestation( ["/approved/exporter", "--dsv41-attest-device", "Metal0"], diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index e7d0fdf3260b..785c194ad011 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Approved processes run in a new POSIX session and process group; the Windows containment implementation creates the process suspended, assigns it to a kill-on-close Job Object, and resumes it only after assignment. A timeout or abnormal execution terminates and reaps the full contained tree before postchecks, and failure to prove quiescence blocks post-attestation. Captured output remains bytes until process-tree cleanup and all lower identity checks finish, then DS4 output is decoded as strict UTF-8 without replacement. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded no-unrelated-child subreaper owner and stable pidfds; the owner kills the exact root, drains every reparented descendant, reaps all owned children, and proves its child set empty without numeric PID or process-group signaling. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. A structured quiescence result gates every post-attestation, and any termination, assignment, reaping, ownership, or quiescence failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when quiescence is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py index 6e714ac0a245..bd6ea82d53fa 100644 --- a/tools/deepseek-v41-trace/run_ds4.py +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -6,6 +6,7 @@ import shlex import subprocess import sys +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -65,6 +66,24 @@ EXPORTER_TRACE_TIMEOUT_SECONDS = 24 * 60 * 60 +@dataclass(frozen=True) +class InvocationSecondaryFailure: + component: str + error: BaseException + + +class InvocationIntegrityError(PreflightError): + def __init__( + self, + message: str, + *, + primary_error: BaseException, + secondary_errors: list[InvocationSecondaryFailure]): + super().__init__(message) + self.primary_error = primary_error + self.secondary_errors = tuple(secondary_errors) + + def git_output(checkout: Path, *args: str) -> str: try: return subprocess.check_output( @@ -202,6 +221,8 @@ def run_exporter_with_post_attestation( exporter_policy: dict[str, Any], expected_runtime_build: dict[str, Any], timeout_seconds: int, + decode_stdout_label: str | None = None, + decode_stderr_label: str | None = None, **kwargs: Any) -> subprocess.CompletedProcess[Any]: result = None primary_error = None @@ -216,10 +237,22 @@ def run_exporter_with_post_attestation( ) except BaseException as error: primary_error = error - if isinstance(primary_error, ExecutionIntegrityError) and any( - failure.component.startswith(("process-tree-", "direct-child-", "containment-")) - for failure in primary_error.secondary_errors): + if primary_error is not None and ( + not isinstance(primary_error, ExecutionIntegrityError) + or not primary_error.quiescence_proven): raise primary_error + if result is not None and primary_error is None: + try: + stdout = ( + decode_exporter_output(result.stdout, label=decode_stdout_label) + if decode_stdout_label is not None else result.stdout) + stderr = ( + decode_exporter_output(result.stderr, label=decode_stderr_label) + if decode_stderr_label is not None else result.stderr) + result = subprocess.CompletedProcess( + result.args, result.returncode, stdout, stderr) + except BaseException as error: + primary_error = error nonzero_error = None if result is not None and result.returncode != 0: nonzero_error = PreflightError(f"{operation} failed: exit {result.returncode}") @@ -237,10 +270,14 @@ def run_exporter_with_post_attestation( reported_primary = primary_error or nonzero_error if reported_primary is not None: if secondary_error is not None: - raise PreflightError( + raise InvocationIntegrityError( f"{operation} primary failure [{type(reported_primary).__name__}: {reported_primary}]; " f"secondary post-invocation runtime-build attestation failure " - f"[{type(secondary_error).__name__}: {secondary_error}]") from reported_primary + f"[{type(secondary_error).__name__}: {secondary_error}]", + primary_error=reported_primary, + secondary_errors=[InvocationSecondaryFailure( + "post-invocation-runtime-build-attestation", secondary_error)], + ) from reported_primary if primary_error is not None: raise primary_error if secondary_error is not None: @@ -267,17 +304,17 @@ def query_accelerator_attestation( timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, check=False, capture_output=True, + decode_stdout_label="selected accelerator query stdout", + decode_stderr_label="selected accelerator query stderr", ) validation_error = None attestation = None if result.returncode != 0: - detail = decode_exporter_output( - result.stderr, label="selected accelerator query stderr").strip() or f"exit {result.returncode}" + detail = result.stderr.strip() or f"exit {result.returncode}" validation_error = PreflightError(f"selected accelerator query failed: {detail}") else: try: - record = strict_json_loads(decode_exporter_output( - result.stdout, label="selected accelerator query stdout")) + record = strict_json_loads(result.stdout) attestation = validate_accelerator_attestation(record, expected_device=device) except (TraceError, PreflightError) as error: validation_error = PreflightError( diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 90347243fc99..f0b4597f2e24 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -13,7 +13,9 @@ import subprocess import sys import tempfile +import threading import time +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any, Iterable @@ -140,6 +142,8 @@ class TraceError(RuntimeError): PROCESS_TREE_TERM_GRACE_SECONDS = 1 WINDOWS_CREATE_SUSPENDED = 0x00000004 WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 +LINUX_PR_SET_CHILD_SUBREAPER = 36 +LINUX_PR_GET_CHILD_SUBREAPER = 37 @dataclass(frozen=True) @@ -154,17 +158,23 @@ def __init__( message: str, *, primary_error: BaseException | None, - secondary_errors: list[_IntegrityFailure]): + secondary_errors: list[_IntegrityFailure], + quiescence_proven: bool = False): super().__init__(message) self.primary_error = primary_error self.secondary_errors = tuple(secondary_errors) + self.quiescence_proven = quiescence_proven @dataclass class _ProcessContainment: process: subprocess.Popen[bytes] - process_group_id: int | None = None + linux_root_pidfd: int | None = None + linux_lock_held: bool = False + test_process_group_id: int | None = None job_handle: int | None = None + windows_job_assigned: bool = False + windows_process_resumed: bool = False @dataclass @@ -173,6 +183,18 @@ class _ContainedRun: primary_error: BaseException | None integrity_failures: list[_IntegrityFailure] containment: _ProcessContainment | None + quiescence_proven: bool = False + process_started: bool = False + + +@dataclass +class _ContainmentCleanup: + failures: list[_IntegrityFailure] + quiescence_proven: bool + + +_LINUX_SUBREAPER_LOCK = threading.Lock() +_TEST_PROCESS_GROUP_CONTAINMENT = threading.local() @dataclass(frozen=True) @@ -798,20 +820,29 @@ def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _P process = subprocess.Popen(command, creationflags=creationflags, **launch) job = None thread = None + job_assigned = False + process_resumed = False try: job = _create_windows_kill_job() kernel32 = _windows_kernel32() process_handle = int(process._handle) if not kernel32.AssignProcessToJobObject(job, process_handle): raise _windows_error("cannot assign suspended process to containment job") + job_assigned = True thread = _open_windows_process_thread(process.pid) if kernel32.ResumeThread(thread) == 0xFFFFFFFF: raise _windows_error("cannot resume contained process") + process_resumed = True thread_closed = kernel32.CloseHandle(thread) thread = None if not thread_closed: raise _windows_error("cannot close resumed process thread") - return _ProcessContainment(process=process, job_handle=job) + return _ProcessContainment( + process=process, + job_handle=job, + windows_job_assigned=job_assigned, + windows_process_resumed=process_resumed, + ) except BaseException as primary_error: kernel32 = _windows_kernel32() failures = [] @@ -820,15 +851,11 @@ def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _P failures.append(_IntegrityFailure( "windows-thread-handle-close", _windows_error("cannot close suspended process thread"))) - if job is not None: + if job_assigned and job is not None: if not kernel32.TerminateJobObject(job, 1): failures.append(_IntegrityFailure( "windows-job-termination", _windows_error("cannot terminate failed process containment job"))) - if not kernel32.CloseHandle(job): - failures.append(_IntegrityFailure( - "windows-job-handle-close", - _windows_error("cannot close failed process containment job"))) else: try: process.kill() @@ -838,6 +865,14 @@ def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _P process.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) except BaseException as error: failures.append(_IntegrityFailure("windows-process-reap", error)) + try: + _close_windows_process_handle(process) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-handle-close", error)) + if job is not None and not kernel32.CloseHandle(job): + failures.append(_IntegrityFailure( + "windows-job-handle-close", + _windows_error("cannot close failed process containment job"))) if failures: raise ExecutionIntegrityError( f"Windows containment startup primary failure " @@ -845,6 +880,189 @@ def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _P f"secondary integrity failures: {_format_integrity_failures(failures)}", primary_error=primary_error, secondary_errors=failures, + quiescence_proven=False, + ) from primary_error + raise ExecutionIntegrityError( + f"Windows containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]", + primary_error=primary_error, + secondary_errors=[], + quiescence_proven=False, + ) from primary_error + + +@contextmanager +def _test_only_process_group_containment() -> Iterable[None]: + previous = getattr(_TEST_PROCESS_GROUP_CONTAINMENT, "enabled", False) + _TEST_PROCESS_GROUP_CONTAINMENT.enabled = True + try: + yield + finally: + _TEST_PROCESS_GROUP_CONTAINMENT.enabled = previous + + +def _linux_direct_children() -> set[int]: + children = set() + task_root = Path("/proc/self/task") + if not task_root.is_dir(): + raise TraceError("Linux subreaper containment requires procfs task children") + for task in task_root.iterdir(): + child_file = task / "children" + try: + values = child_file.read_text(encoding="ascii").split() + except FileNotFoundError: + continue + except OSError as error: + raise TraceError(f"cannot read Linux subreaper child ownership: {error}") from error + for value in values: + try: + children.add(int(value)) + except ValueError as error: + raise TraceError("Linux subreaper child ownership is invalid") from error + return children + + +def _linux_task_ids() -> set[int]: + task_root = Path("/proc/self/task") + if not task_root.is_dir(): + raise TraceError("Linux subreaper containment requires procfs task identities") + try: + return {int(task.name) for task in task_root.iterdir()} + except (OSError, ValueError) as error: + raise TraceError(f"cannot read Linux subreaper task identities: {error}") from error + + +def _linux_enable_child_subreaper() -> None: + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(LINUX_PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) != 0: + error_number = ctypes.get_errno() + raise TraceError(f"cannot enable Linux child subreaper: {os.strerror(error_number)}") + enabled = ctypes.c_int() + if libc.prctl(LINUX_PR_GET_CHILD_SUBREAPER, ctypes.byref(enabled), 0, 0, 0) != 0: + error_number = ctypes.get_errno() + raise TraceError(f"cannot verify Linux child subreaper: {os.strerror(error_number)}") + if enabled.value != 1: + raise TraceError("Linux child subreaper did not remain enabled") + + +def _linux_open_pidfd(process_id: int) -> int: + opener = getattr(os, "pidfd_open", None) + sender = getattr(signal, "pidfd_send_signal", None) + if opener is None or sender is None: + raise TraceError("Linux subreaper containment requires pidfd signaling") + try: + return int(opener(process_id, 0)) + except ProcessLookupError: + raise + except OSError as error: + raise TraceError(f"cannot open stable Linux process identity: {error}") from error + + +def _linux_require_pidfd_support() -> None: + if getattr(os, "pidfd_open", None) is None or getattr(signal, "pidfd_send_signal", None) is None: + raise TraceError("Linux subreaper containment requires pidfd signaling") + + +def _linux_signal_pidfd(pidfd: int, requested_signal: int) -> None: + sender = getattr(signal, "pidfd_send_signal", None) + if sender is None: + raise TraceError("Linux subreaper containment requires pidfd signaling") + sender(pidfd, requested_signal) + + +def _linux_reap_owned_descendants(root_pid: int) -> list[_IntegrityFailure]: + failures = [] + for process_id in sorted(_linux_direct_children()): + if process_id == root_pid: + continue + try: + os.waitpid(process_id, os.WNOHANG) + except ChildProcessError: + continue + except BaseException as error: + failures.append(_IntegrityFailure("linux-descendant-reap", error)) + return failures + + +def _linux_signal_owned_children( + containment: _ProcessContainment, + requested_signal: int, +) -> list[_IntegrityFailure]: + failures = [] + root_pid = containment.process.pid + try: + process_ids = _linux_direct_children() + except BaseException as error: + return [_IntegrityFailure("linux-child-ownership", error)] + for process_id in sorted(process_ids): + pidfd = containment.linux_root_pidfd if process_id == root_pid else None + close_pidfd = False + try: + if pidfd is None: + pidfd = _linux_open_pidfd(process_id) + close_pidfd = True + _linux_signal_pidfd(pidfd, requested_signal) + except ProcessLookupError: + pass + except BaseException as error: + failures.append(_IntegrityFailure("linux-process-termination", error)) + finally: + if close_pidfd and pidfd is not None: + try: + os.close(pidfd) + except BaseException as error: + failures.append(_IntegrityFailure("linux-pidfd-close", error)) + return failures + + +def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + if not _LINUX_SUBREAPER_LOCK.acquire(blocking=False): + raise TraceError("Linux subreaper containment is already active") + process = None + pidfd = None + try: + _linux_enable_child_subreaper() + _linux_require_pidfd_support() + if len(_linux_task_ids()) != 1: + raise TraceError("Linux subreaper containment requires a single-threaded supervisor") + if _linux_direct_children(): + raise TraceError("Linux subreaper containment process owns unrelated children") + process = subprocess.Popen(command, **launch) + pidfd = _linux_open_pidfd(process.pid) + return _ProcessContainment( + process=process, + linux_root_pidfd=pidfd, + linux_lock_held=True, + ) + except BaseException as primary_error: + failures = [] + if process is not None: + failed_containment = _ProcessContainment( + process=process, + linux_root_pidfd=pidfd, + linux_lock_held=True, + ) + cleanup = _terminate_process_tree(failed_containment) + failures.extend(cleanup.failures) + failures.extend(_close_process_containment(failed_containment)) + else: + _LINUX_SUBREAPER_LOCK.release() + if failures: + raise ExecutionIntegrityError( + f"Linux containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: {_format_integrity_failures(failures)}", + primary_error=primary_error, + secondary_errors=failures, + quiescence_proven=False, + ) from primary_error + if process is not None: + raise ExecutionIntegrityError( + f"Linux containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]", + primary_error=primary_error, + secondary_errors=[], + quiescence_proven=False, ) from primary_error raise @@ -852,12 +1070,16 @@ def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _P def _start_contained_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: if sys.platform == "win32": return _start_windows_job_process(command, launch) - launch["start_new_session"] = True - process = subprocess.Popen(command, **launch) - return _ProcessContainment(process=process, process_group_id=process.pid) + if sys.platform == "linux": + return _start_linux_subreaper_process(command, launch) + if sys.platform == "darwin" and getattr(_TEST_PROCESS_GROUP_CONTAINMENT, "enabled", False): + launch["start_new_session"] = True + process = subprocess.Popen(command, **launch) + return _ProcessContainment(process=process, test_process_group_id=process.pid) + raise TraceError("proven process containment is unavailable on this platform") -def _posix_process_group_exists(process_group_id: int) -> bool: +def _test_process_group_exists(process_group_id: int) -> bool: try: os.killpg(process_group_id, 0) return True @@ -890,12 +1112,26 @@ class BasicAccountingInformation(ctypes.Structure): return int(accounting.ActiveProcesses) +def _close_windows_process_handle(process: subprocess.Popen[bytes]) -> None: + process_handle = getattr(process, "_handle", None) + if process_handle is None: + return + if not _windows_kernel32().CloseHandle(int(process_handle)): + raise _windows_error("cannot close process handle") + process._handle = None + + def _process_tree_is_quiescent(containment: _ProcessContainment) -> bool: if containment.job_handle is not None: return _windows_job_active_processes(containment.job_handle) == 0 - if containment.process_group_id is None: - raise TraceError("process containment identity is missing") - return not _posix_process_group_exists(containment.process_group_id) + if containment.linux_lock_held: + reap_failures = _linux_reap_owned_descendants(containment.process.pid) + if reap_failures: + raise reap_failures[0].error + return containment.process.poll() is not None and not _linux_direct_children() + if containment.test_process_group_id is not None: + return not _test_process_group_exists(containment.test_process_group_id) + raise TraceError("process containment identity is missing") def _wait_for_process_tree_quiescence(containment: _ProcessContainment, deadline: float) -> None: @@ -905,42 +1141,86 @@ def _wait_for_process_tree_quiescence(containment: _ProcessContainment, deadline time.sleep(0.01) -def _terminate_process_tree(containment: _ProcessContainment) -> list[_IntegrityFailure]: +def _terminate_process_tree(containment: _ProcessContainment) -> _ContainmentCleanup: failures = [] deadline = time.monotonic() + PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS process = containment.process - try: - if containment.job_handle is not None: + if containment.job_handle is not None: + try: if not _windows_kernel32().TerminateJobObject(containment.job_handle, 1): raise _windows_error("cannot terminate process containment job") - elif containment.process_group_id is not None and _posix_process_group_exists(containment.process_group_id): - try: - os.killpg(containment.process_group_id, signal.SIGTERM) - except (ProcessLookupError, PermissionError): - pass - term_deadline = min(deadline, time.monotonic() + PROCESS_TREE_TERM_GRACE_SECONDS) + except BaseException as error: + failures.append(_IntegrityFailure("windows-job-termination", error)) + try: + process.communicate(timeout=max(0.01, deadline - time.monotonic())) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-reap", error)) + elif containment.linux_lock_held: + failures.extend(_linux_signal_owned_children(containment, signal.SIGTERM)) + try: + process.communicate(timeout=PROCESS_TREE_TERM_GRACE_SECONDS) + except subprocess.TimeoutExpired: + failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) try: - _wait_for_process_tree_quiescence(containment, term_deadline) - except TraceError: - if _posix_process_group_exists(containment.process_group_id): - try: - os.killpg(containment.process_group_id, signal.SIGKILL) - except (ProcessLookupError, PermissionError): - pass - else: - raise TraceError("process containment identity is missing") - except BaseException as error: - failures.append(_IntegrityFailure("process-tree-termination", error)) - try: - remaining = max(0.01, deadline - time.monotonic()) - process.communicate(timeout=remaining) - except BaseException as error: - failures.append(_IntegrityFailure("direct-child-reap", error)) + process.communicate(timeout=max(0.01, deadline - time.monotonic())) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + descendant_term_deadline = min( + deadline, time.monotonic() + PROCESS_TREE_TERM_GRACE_SECONDS) + failures.extend(_linux_signal_owned_children(containment, signal.SIGTERM)) + try: + _wait_for_process_tree_quiescence(containment, descendant_term_deadline) + except BaseException: + failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) + while time.monotonic() < deadline: + failures.extend(_linux_reap_owned_descendants(process.pid)) + try: + if _process_tree_is_quiescent(containment): + break + except BaseException as error: + failures.append(_IntegrityFailure("linux-child-ownership", error)) + break + failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) + time.sleep(0.01) + elif containment.test_process_group_id is not None: + try: + if _test_process_group_exists(containment.test_process_group_id): + try: + os.killpg(containment.test_process_group_id, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + term_deadline = min(deadline, time.monotonic() + PROCESS_TREE_TERM_GRACE_SECONDS) + try: + _wait_for_process_tree_quiescence(containment, term_deadline) + except TraceError: + if _test_process_group_exists(containment.test_process_group_id): + try: + os.killpg(containment.test_process_group_id, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-termination", error)) + try: + process.communicate(timeout=max(0.01, deadline - time.monotonic())) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + else: + failures.append(_IntegrityFailure( + "process-tree-termination", TraceError("process containment identity is missing"))) try: _wait_for_process_tree_quiescence(containment, deadline) except BaseException as error: failures.append(_IntegrityFailure("process-tree-quiescence", error)) - return failures + quiescence_proven = not failures + if quiescence_proven: + try: + quiescence_proven = _process_tree_is_quiescent(containment) + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-quiescence", error)) + quiescence_proven = False + return _ContainmentCleanup(failures, quiescence_proven) def _run_contained_process( @@ -960,38 +1240,58 @@ def _run_contained_process( error.primary_error or error, list(error.secondary_errors), None, + error.quiescence_proven, + True, ) except BaseException as error: - return _ContainedRun(None, error, [], None) + return _ContainedRun(None, error, [], None, False, False) try: stdout, stderr = containment.process.communicate(input=input_data, timeout=timeout) result = subprocess.CompletedProcess( command, containment.process.returncode, stdout, stderr) except BaseException as error: - failures = _terminate_process_tree(containment) - return _ContainedRun(None, error, failures, containment) + cleanup = _terminate_process_tree(containment) + return _ContainedRun( + None, error, cleanup.failures, containment, cleanup.quiescence_proven, True) try: if _process_tree_is_quiescent(containment): - return _ContainedRun(result, None, [], containment) + return _ContainedRun(result, None, [], containment, True, True) except BaseException as error: failures = [_IntegrityFailure("process-tree-quiescence", error)] else: error = TraceError(f"{label} process tree remained active after direct child exit") failures = [] - failures.extend(_terminate_process_tree(containment)) - return _ContainedRun(None, error, failures, containment) + cleanup = _terminate_process_tree(containment) + failures.extend(cleanup.failures) + return _ContainedRun( + None, error, failures, containment, cleanup.quiescence_proven, True) def _close_process_containment(containment: _ProcessContainment | None) -> list[_IntegrityFailure]: - if containment is None or containment.job_handle is None: - return [] - try: - if not _windows_kernel32().CloseHandle(containment.job_handle): - raise _windows_error("cannot close process containment job") - containment.job_handle = None + if containment is None: return [] - except BaseException as error: - return [_IntegrityFailure("containment-handle-close", error)] + failures = [] + if containment.job_handle is not None: + try: + if not _windows_kernel32().CloseHandle(containment.job_handle): + raise _windows_error("cannot close process containment job") + containment.job_handle = None + except BaseException as error: + failures.append(_IntegrityFailure("containment-handle-close", error)) + try: + _close_windows_process_handle(containment.process) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-handle-close", error)) + if containment.linux_root_pidfd is not None: + try: + os.close(containment.linux_root_pidfd) + containment.linux_root_pidfd = None + except BaseException as error: + failures.append(_IntegrityFailure("linux-root-pidfd-close", error)) + if containment.linux_lock_held: + containment.linux_lock_held = False + _LINUX_SUBREAPER_LOCK.release() + return failures def _format_integrity_failures(failures: list[_IntegrityFailure]) -> str: @@ -1005,23 +1305,27 @@ def _raise_execution_integrity_failures( label: str, primary_error: BaseException | None, integrity_failures: list[_IntegrityFailure], + containment_started: bool, + quiescence_proven: bool, ) -> None: - if primary_error is not None and integrity_failures: + if primary_error is not None and containment_started: + suffix = "" + if integrity_failures: + suffix = f"; secondary integrity failures: {_format_integrity_failures(integrity_failures)}" raise ExecutionIntegrityError( - f"{label} primary failure [{type(primary_error).__name__}: {primary_error}]; " - f"secondary integrity failures: {_format_integrity_failures(integrity_failures)}", + f"{label} primary failure [{type(primary_error).__name__}: {primary_error}]{suffix}", primary_error=primary_error, secondary_errors=integrity_failures, + quiescence_proven=quiescence_proven, ) from primary_error if primary_error is not None: raise primary_error - if len(integrity_failures) == 1: - raise integrity_failures[0].error if integrity_failures: raise ExecutionIntegrityError( f"{label} integrity failures: {_format_integrity_failures(integrity_failures)}", primary_error=None, secondary_errors=integrity_failures, + quiescence_proven=quiescence_proven, ) from integrity_failures[0].error @@ -1091,8 +1395,11 @@ def run_approved_executable( runtime_files: list[tuple[ExecutableFileReceipt, int]] = [] containment = None result = None + decoded_result = None primary_error = None integrity_failures: list[_IntegrityFailure] = [] + containment_started = False + quiescence_proven = False try: for component in runtime_policy["runtime_receipt"]["components"]: runtime_path = Path(runtime_policy["install_root"]) / "lib" / component["filename"] @@ -1126,6 +1433,8 @@ def run_approved_executable( launch=launch, ) containment = contained.containment + containment_started = contained.process_started + quiescence_proven = contained.quiescence_proven result = contained.result primary_error = contained.primary_error integrity_failures.extend(contained.integrity_failures) @@ -1209,18 +1518,25 @@ def run_approved_executable( except BaseException as error: integrity_failures.append(_IntegrityFailure("executable-descriptor-close", error)) integrity_failures.extend(_close_process_containment(containment)) + if result is not None and primary_error is None: + try: + stdout = _decode_subprocess_stream( + result.stdout, text=text, encoding=encoding, errors=errors) + stderr = _decode_subprocess_stream( + result.stderr, text=text, encoding=encoding, errors=errors) + decoded_result = subprocess.CompletedProcess( + result.args, result.returncode, stdout, stderr) + except BaseException as error: + primary_error = error _raise_execution_integrity_failures( label=label, primary_error=primary_error, integrity_failures=integrity_failures, + containment_started=containment_started, + quiescence_proven=quiescence_proven, ) - if result is None: + if decoded_result is None: raise TraceError(f"{label} execution did not return a result") - stdout = _decode_subprocess_stream( - result.stdout, text=text, encoding=encoding, errors=errors) - stderr = _decode_subprocess_stream( - result.stderr, text=text, encoding=encoding, errors=errors) - decoded_result = subprocess.CompletedProcess(result.args, result.returncode, stdout, stderr) return decoded_result, identity From a34c30c6a709e3890d40c82b5ab86c9ada5f4ec9 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 15:35:11 -0700 Subject: [PATCH 41/56] trace : harden approved process containment Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 369 ++++++++++++++++++- tools/deepseek-v41-trace/README.md | 2 +- tools/deepseek-v41-trace/trace_format.py | 441 ++++++++++++++++++++++- 3 files changed, 776 insertions(+), 36 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 2dd2ccdd4424..db123248176d 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2401,22 +2401,26 @@ def test_linux_subreaper_rejects_unrelated_child_ownership(self) -> None: lock = mock.Mock() lock.acquire.return_value = True with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_linux_enable_child_subreaper"), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( + trace, "_linux_set_child_subreaper") as set_subreaper, mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_linux_direct_children", return_value={42}), mock.patch.object( - trace.subprocess, "Popen") as popen, self.assertRaisesRegex( + trace, "_start_linux_blocked_process") as start, self.assertRaisesRegex( trace.TraceError, "owns unrelated children"): trace._start_linux_subreaper_process(["approved"], {}) - popen.assert_not_called() + start.assert_not_called() + self.assertEqual(set_subreaper.call_args_list, [mock.call(1), mock.call(0)]) lock.release.assert_called_once() def test_linux_subreaper_boundary_precedes_target_execution(self) -> None: source = inspect.getsource(trace._start_linux_subreaper_process) - self.assertLess(source.index("_linux_enable_child_subreaper"), source.index("subprocess.Popen")) - self.assertLess(source.index("_linux_task_ids"), source.index("subprocess.Popen")) - self.assertLess(source.index("_linux_direct_children"), source.index("subprocess.Popen")) - self.assertIn("_linux_open_pidfd", source) + self.assertLess(source.index("_linux_get_child_subreaper"), source.index("_start_linux_blocked_process")) + self.assertLess(source.index("_linux_task_ids"), source.index("_start_linux_blocked_process")) + self.assertLess(source.index("_linux_direct_children"), source.index("_start_linux_blocked_process")) + self.assertLess(source.index("_start_linux_blocked_process"), source.index("_linux_open_pidfd")) + self.assertLess(source.index("_linux_open_pidfd"), source.index("process.release_exec")) self.assertNotIn("killpg", source) self.assertNotIn("os.kill(", inspect.getsource(trace._linux_signal_owned_children)) @@ -2424,14 +2428,174 @@ def test_linux_subreaper_without_pidfd_support_fails_before_launch(self) -> None lock = mock.Mock() lock.acquire.return_value = True with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_linux_enable_child_subreaper"), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( + trace, "_linux_set_child_subreaper") as set_subreaper, mock.patch.object( trace, "_linux_require_pidfd_support", side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( - trace.subprocess, "Popen") as popen, self.assertRaisesRegex( + trace, "_start_linux_blocked_process") as start, self.assertRaisesRegex( trace.TraceError, "pidfd unavailable"): trace._start_linux_subreaper_process(["approved"], {}) - popen.assert_not_called() + start.assert_not_called() + self.assertEqual(set_subreaper.call_args_list, [mock.call(1), mock.call(0)]) lock.release.assert_called_once() + def test_linux_prelaunch_primary_and_restore_failure_are_both_retained(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + primary = trace.TraceError("pidfd unavailable") + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( + trace, "_linux_set_child_subreaper", side_effect=[None, OSError("restore failed")]), mock.patch.object( + trace, "_linux_require_pidfd_support", side_effect=primary), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_subreaper_process(["approved"], {}) + self.assertIs(raised.exception.primary_error, primary) + self.assertIs(raised.exception.__cause__, primary) + self.assertFalse(raised.exception.quiescence_proven) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-subreaper-restore"], + ) + lock.release.assert_called_once() + + def test_linux_pidfd_failure_aborts_blocked_child_before_exec(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + process = mock.Mock(pid=71) + process.abort_blocked.return_value = trace._ContainmentCleanup([], True) + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( + trace, "_linux_set_child_subreaper") as set_subreaper, mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_linux_direct_children", return_value=set()), mock.patch.object( + trace, "_start_linux_blocked_process", return_value=process), mock.patch.object( + trace, "_linux_open_pidfd", side_effect=OSError("pidfd failed")), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_subreaper_process(["approved"], {}) + self.assertFalse(raised.exception.quiescence_proven) + process.abort_blocked.assert_called_once() + process.release_exec.assert_not_called() + self.assertEqual(set_subreaper.call_args_list, [mock.call(1), mock.call(0)]) + lock.release.assert_called_once() + + def test_linux_exec_barrier_releases_only_after_pidfd(self) -> None: + events = [] + lock = mock.Mock() + lock.acquire.return_value = True + process = mock.Mock(pid=71) + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_get_child_subreaper", return_value=1), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_linux_direct_children", return_value=set()), mock.patch.object( + trace, "_start_linux_blocked_process", + side_effect=lambda *_args: events.append("blocked") or process), mock.patch.object( + trace, "_linux_open_pidfd", + side_effect=lambda pid: events.append(f"pidfd:{pid}") or 90), mock.patch.object( + trace.os, "close"), mock.patch.object( + trace, "_linux_set_child_subreaper") as restore: + process.release_exec.side_effect = lambda: events.append("exec") + containment = trace._start_linux_subreaper_process(["approved"], {}) + self.assertEqual(containment.linux_root_pidfd, 90) + self.assertEqual(containment.linux_prior_subreaper, 1) + self.assertTrue(containment.linux_exec_released) + self.assertEqual(trace._close_process_containment( + containment, quiescence_proven=True), []) + self.assertEqual(events, ["blocked", "pidfd:71", "exec"]) + restore.assert_called_once_with(1) + lock.release.assert_called_once() + + def test_linux_blocked_child_never_signals_after_identity_loss(self) -> None: + process = trace._LinuxForkExecProcess( + ["approved"], + 71, + barrier_fd=80, + exec_error_fd=81, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object(trace.os, "waitpid", side_effect=ChildProcessError), mock.patch.object( + trace.os, "kill") as kill, self.assertRaisesRegex( + trace.TraceError, "identity was lost"): + process.kill() + kill.assert_not_called() + + @unittest.skipUnless(sys.platform == "linux", "Linux fork/exec containment test") + def test_linux_fork_exec_barrier_blocks_target_until_release(self) -> None: + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp) / "target-ran" + process = trace._start_linux_blocked_process( + ["/bin/sh", "-c", "printf ran > \"$1\"", "sh", str(marker)], + {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE}, + ) + self.assertFalse(marker.exists()) + process.release_exec() + stdout, stderr = process.communicate(timeout=5) + self.assertEqual(process.returncode, 0) + self.assertEqual(stdout, b"") + self.assertEqual(stderr, b"") + self.assertEqual(marker.read_text(encoding="ascii"), "ran") + self.assertEqual(process.close_streams(), []) + + @unittest.skipUnless(sys.platform == "linux", "Linux fork/exec containment test") + def test_linux_blocked_child_abort_reaps_without_target_execution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp) / "target-ran" + before = len(os.listdir("/proc/self/fd")) + process = trace._start_linux_blocked_process( + ["/bin/sh", "-c", "printf ran > \"$1\"", "sh", str(marker)], + {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE}, + ) + cleanup = process.abort_blocked() + self.assertTrue(cleanup.quiescence_proven) + self.assertEqual(cleanup.failures, []) + self.assertFalse(marker.exists()) + self.assertEqual(process.close_streams(), []) + self.assertEqual(len(os.listdir("/proc/self/fd")), before) + + def test_linux_process_stream_close_reports_every_failure(self) -> None: + process = trace._LinuxForkExecProcess( + ["approved"], + 71, + barrier_fd=80, + exec_error_fd=81, + stdin_fd=82, + stdout_fd=83, + stderr_fd=84, + ) + with mock.patch.object(trace.os, "close", side_effect=[ + OSError("barrier"), None, OSError("stdin"), None, OSError("stderr")]): + failures = process.close_streams() + self.assertEqual( + [failure.component for failure in failures], + [ + "linux-process-fd-close:barrier", + "linux-process-fd-close:stdin", + "linux-process-fd-close:stderr", + ], + ) + self.assertEqual( + [str(failure.error) for failure in failures], + ["barrier", "stdin", "stderr"], + ) + + def test_linux_fork_exec_rejects_controls_without_descriptor_leaks(self) -> None: + for launch in ( + {"shell": True}, + {"close_fds": False}, + {"stdin": subprocess.PIPE, "stderr": object()}, + ): + with self.subTest(controls=sorted(launch)): + before = len(os.listdir("/dev/fd")) + with self.assertRaisesRegex(trace.TraceError, "unsupported"): + trace._start_linux_blocked_process(["/bin/true"], launch) + self.assertEqual(len(os.listdir("/dev/fd")), before) + def test_unproven_posix_containment_fails_closed_before_setsid_escape(self) -> None: if sys.platform == "linux": self.skipTest("Linux uses subreaper and pidfd containment") @@ -2522,8 +2686,22 @@ def test_windows_job_assignment_happens_before_resume(self) -> None: self.assertTrue(containment.windows_process_resumed) def test_windows_job_assignment_failure_kills_and_reaps_exact_child(self) -> None: + class OwnedHandle: + def __init__(self, value: int): + self.value = value + self.close_count = 0 + + def __int__(self) -> int: + return self.value + + def Close(self) -> None: + if self.close_count: + raise AssertionError("process handle closed twice") + self.close_count += 1 + process = mock.Mock(pid=91) - process._handle = 92 + process._handle = OwnedHandle(92) + owned_handle = process._handle kernel32 = mock.Mock() kernel32.AssignProcessToJobObject.return_value = 0 kernel32.CloseHandle.return_value = 1 @@ -2537,11 +2715,167 @@ def test_windows_job_assignment_failure_kills_and_reaps_exact_child(self) -> Non process.kill.assert_called_once() process.wait.assert_called_once_with(timeout=trace.PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) kernel32.TerminateJobObject.assert_not_called() + self.assertEqual(kernel32.CloseHandle.call_args_list, [mock.call(93)]) + self.assertEqual(owned_handle.close_count, 1) + self.assertIsNone(process._handle) + + def test_windows_process_handle_ownership_closes_once(self) -> None: + class OwnedHandle: + def __init__(self): + self.close_count = 0 + + def Close(self) -> None: + if self.close_count: + raise AssertionError("recycled process handle closed") + self.close_count += 1 + + process = mock.Mock() + owned_handle = OwnedHandle() + process._handle = owned_handle + with mock.patch.object(trace, "_windows_kernel32") as kernel32: + trace._close_windows_process_handle(process) + trace._close_windows_process_handle(process) + self.assertEqual(owned_handle.close_count, 1) + self.assertIsNone(process._handle) + kernel32.assert_not_called() + + def test_linux_subreaper_restores_exact_prior_state(self) -> None: + for prior_state in (0, 1): + with self.subTest(prior_state=prior_state): + lock = mock.Mock() + containment = trace._ProcessContainment( + process=mock.Mock(), + linux_lock_held=True, + linux_prior_subreaper=prior_state, + ) + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_linux_set_child_subreaper") as restore: + failures = trace._close_process_containment( + containment, quiescence_proven=True) + self.assertEqual(failures, []) + restore.assert_called_once_with(prior_state) + lock.release.assert_called_once() + + def test_linux_subreaper_restore_failure_poisoning_blocks_completion(self) -> None: + lock = mock.Mock() + containment = trace._ProcessContainment( + process=mock.Mock(), + linux_lock_held=True, + linux_prior_subreaper=0, + ) + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_set_child_subreaper", side_effect=OSError("restore failed")): + failures = trace._close_process_containment( + containment, quiescence_proven=True) + self.assertTrue(trace._LINUX_SUBREAPER_POISONED) + self.assertEqual([failure.component for failure in failures], ["linux-subreaper-restore"]) + lock.release.assert_called_once() + + def test_linux_subreaper_restores_after_timeout_and_target_exception(self) -> None: + for primary in ( + subprocess.TimeoutExpired(["approved"], 1), + OSError("target failed"), + ): + with self.subTest(primary=type(primary).__name__): + lock = mock.Mock() + process = mock.Mock(pid=71) + process.communicate.side_effect = primary + containment = trace._ProcessContainment( + process=process, + linux_root_pidfd=90, + linux_lock_held=True, + linux_prior_subreaper=0, + linux_exec_released=True, + ) + cleanup = trace._ContainmentCleanup([], True) + with mock.patch.object( + trace, "_start_contained_process", return_value=containment), mock.patch.object( + trace, "_terminate_process_tree", return_value=cleanup), mock.patch.object( + trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace.os, "close"), mock.patch.object( + trace, "_linux_set_child_subreaper") as restore: + result = trace._run_contained_process( + ["approved"], + label="approved executable", + timeout=1, + input_data=None, + launch={}, + ) + close_failures = trace._close_process_containment( + containment, + quiescence_proven=result.quiescence_proven, + ) + self.assertIs(result.primary_error, primary) + self.assertEqual(close_failures, []) + restore.assert_called_once_with(0) + lock.release.assert_called_once() + + def test_linux_subreaper_does_not_restore_without_tree_quiescence(self) -> None: + lock = mock.Mock() + containment = trace._ProcessContainment( + process=mock.Mock(), + linux_lock_held=True, + linux_prior_subreaper=0, + ) + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( + trace, "_linux_set_child_subreaper") as restore: + failures = trace._close_process_containment( + containment, + quiescence_proven=False, + ) + self.assertTrue(trace._LINUX_SUBREAPER_POISONED) + restore.assert_not_called() + self.assertEqual([failure.component for failure in failures], ["linux-subreaper-restore"]) + lock.release.assert_called_once() + + def test_linux_poisoned_subreaper_supervisor_rejects_reuse(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_SUBREAPER_POISONED", True), mock.patch.object( + trace, "_start_linux_blocked_process") as start, self.assertRaisesRegex( + trace.TraceError, "not reusable"): + trace._start_linux_subreaper_process(["approved"], {}) + start.assert_not_called() + lock.release.assert_called_once() + + def test_containment_handle_close_failure_forces_quiescence_false(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"approved") + executable.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": {"components": []}, + } + result = subprocess.CompletedProcess([str(executable)], 0, b"", b"") + contained = trace._ContainedRun(result, None, [], mock.Mock(), True, True) + close_failure = trace._IntegrityFailure( + "linux-root-pidfd-close", OSError("close failed")) + with isolated_test_install_trust(), mock.patch.object( + trace, "_run_contained_process", return_value=contained), mock.patch.object( + trace, "_close_process_containment", return_value=[close_failure]), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace.run_approved_executable( + [str(executable)], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + check=False, + capture_output=True, + ) + self.assertFalse(raised.exception.quiescence_proven) self.assertEqual( - kernel32.CloseHandle.call_args_list, - [mock.call(92), mock.call(93)], + [failure.component for failure in raised.exception.secondary_errors], + ["linux-root-pidfd-close"], ) - self.assertIsNone(process._handle) def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -3682,7 +4016,12 @@ def test_ds4_explicit_quiescence_false_always_blocks_post_attestation(self) -> N "windows-process-reap", "windows-process-termination", "windows-job-assignment", - "linux-child-ownership"): + "linux-child-ownership", + "linux-root-pidfd-close", + "linux-subreaper-restore", + "windows-process-handle-close", + "windows-thread-handle-close", + "containment-handle-close"): with self.subTest(component=component): primary = subprocess.TimeoutExpired(["exporter"], 7) failure = runtime_trace._IntegrityFailure( diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 785c194ad011..1f4a4bb751b5 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded no-unrelated-child subreaper owner and stable pidfds; the owner kills the exact root, drains every reparented descendant, reaps all owned children, and proves its child set empty without numeric PID or process-group signaling. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. A structured quiescence result gates every post-attestation, and any termination, assignment, reaping, ownership, or quiescence failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when quiescence is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded no-unrelated-child subreaper owner and stable pidfds. The launcher snapshots the exact prior child-subreaper state, forks a trusted bootstrap that cannot execute target code, acquires the root pidfd while that child remains behind a one-shot exec barrier, and only then releases the target. A failed pidfd acquisition kills and reaps the exact blocked child before the barrier can release. After execution, the owner kills the exact root, drains every reparented descendant, reaps all owned children, proves its child set empty without numeric PID or process-group signaling, restores and verifies the exact prior subreaper state, and closes every containment handle before releasing the ownership lock. A restoration failure poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, termination, assignment, reaping, empty-set proof, subreaper restoration, or pidfd, Job, process, or thread handle teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index f0b4597f2e24..c49090181710 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -7,6 +7,7 @@ import math import os import re +import selectors import signal import stat import struct @@ -168,9 +169,11 @@ def __init__( @dataclass class _ProcessContainment: - process: subprocess.Popen[bytes] + process: Any linux_root_pidfd: int | None = None linux_lock_held: bool = False + linux_prior_subreaper: int | None = None + linux_exec_released: bool = False test_process_group_id: int | None = None job_handle: int | None = None windows_job_assigned: bool = False @@ -194,6 +197,7 @@ class _ContainmentCleanup: _LINUX_SUBREAPER_LOCK = threading.Lock() +_LINUX_SUBREAPER_POISONED = False _TEST_PROCESS_GROUP_CONTAINMENT = threading.local() @@ -901,6 +905,324 @@ def _test_only_process_group_containment() -> Iterable[None]: _TEST_PROCESS_GROUP_CONTAINMENT.enabled = previous +class _LinuxForkExecProcess: + def __init__( + self, + command: list[str], + pid: int, + *, + barrier_fd: int, + exec_error_fd: int, + stdin_fd: int | None, + stdout_fd: int | None, + stderr_fd: int | None): + self.args = command + self.pid = pid + self.returncode = None + self._barrier_fd = barrier_fd + self._exec_error_fd = exec_error_fd + self._stdin_fd = stdin_fd + self._stdout_fd = stdout_fd + self._stderr_fd = stderr_fd + + def release_exec(self) -> None: + try: + os.write(self._barrier_fd, b"1") + finally: + os.close(self._barrier_fd) + self._barrier_fd = -1 + error_bytes = bytearray() + try: + while True: + chunk = os.read(self._exec_error_fd, 4096) + if not chunk: + break + error_bytes.extend(chunk) + finally: + os.close(self._exec_error_fd) + self._exec_error_fd = -1 + if error_bytes: + self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + raise OSError(error_bytes.decode("ascii", "strict")) + + def abort_blocked(self) -> _ContainmentCleanup: + failures = [] + if self._barrier_fd >= 0: + try: + os.close(self._barrier_fd) + self._barrier_fd = -1 + except BaseException as error: + failures.append(_IntegrityFailure("linux-exec-barrier-close", error)) + try: + self.kill() + except BaseException as error: + failures.append(_IntegrityFailure("linux-blocked-child-termination", error)) + try: + self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + except BaseException as error: + failures.append(_IntegrityFailure("linux-blocked-child-reap", error)) + return _ContainmentCleanup(failures, not failures and self.returncode is not None) + + def close_streams(self) -> list[_IntegrityFailure]: + failures = [] + for attribute in ( + "_barrier_fd", "_exec_error_fd", "_stdin_fd", "_stdout_fd", "_stderr_fd"): + descriptor = getattr(self, attribute) + if descriptor is None or descriptor < 0: + continue + try: + os.close(descriptor) + except BaseException as error: + failures.append(_IntegrityFailure( + f"linux-process-fd-close:{attribute.removeprefix('_').removesuffix('_fd')}", + error, + )) + setattr(self, attribute, -1 if attribute in {"_barrier_fd", "_exec_error_fd"} else None) + return failures + + def poll(self) -> int | None: + if self.returncode is not None: + return self.returncode + try: + waited_pid, status = os.waitpid(self.pid, os.WNOHANG) + except ChildProcessError as error: + if self.returncode is None: + raise TraceError("owned Linux child identity was lost before reap") from error + return self.returncode + if waited_pid == 0: + return None + self.returncode = os.waitstatus_to_exitcode(status) + return self.returncode + + def wait(self, timeout: float | None = None) -> int: + deadline = None if timeout is None else time.monotonic() + timeout + while self.poll() is None: + if deadline is not None and time.monotonic() >= deadline: + raise subprocess.TimeoutExpired(self.args, timeout) + time.sleep(0.01) + return int(self.returncode) + + def kill(self) -> None: + if self.poll() is None: + os.kill(self.pid, signal.SIGKILL) + + def communicate( + self, + input: bytes | None = None, + timeout: float | None = None, + ) -> tuple[bytes | None, bytes | None]: + if input is not None and self._stdin_fd is None: + raise ValueError("stdin is not a pipe") + output = bytearray() + errors = bytearray() + had_stdout = self._stdout_fd is not None + had_stderr = self._stderr_fd is not None + selector = selectors.DefaultSelector() + streams = {} + input_view = memoryview(input or b"") + input_offset = 0 + if self._stdin_fd is not None: + if input is None: + os.close(self._stdin_fd) + self._stdin_fd = None + else: + os.set_blocking(self._stdin_fd, False) + selector.register(self._stdin_fd, selectors.EVENT_WRITE) + for fd, buffer in ((self._stdout_fd, output), (self._stderr_fd, errors)): + if fd is not None: + os.set_blocking(fd, False) + selector.register(fd, selectors.EVENT_READ) + streams[fd] = buffer + deadline = None if timeout is None else time.monotonic() + timeout + try: + while selector.get_map() or self.poll() is None: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired( + self.args, + timeout, + output=bytes(output) if self._stdout_fd is not None else None, + stderr=bytes(errors) if self._stderr_fd is not None else None, + ) + else: + remaining = 0.05 + for key, events in selector.select(min(remaining, 0.05)): + if key.fd == self._stdin_fd and events & selectors.EVENT_WRITE: + try: + written = os.write( + key.fd, input_view[input_offset:input_offset + 65536]) + input_offset += written + except BrokenPipeError: + input_offset = len(input_view) + if input_offset == len(input_view): + selector.unregister(key.fd) + os.close(key.fd) + self._stdin_fd = None + continue + chunk = os.read(key.fd, 65536) + if chunk: + streams[key.fd].extend(chunk) + else: + selector.unregister(key.fd) + os.close(key.fd) + if key.fd == self._stdout_fd: + self._stdout_fd = None + if key.fd == self._stderr_fd: + self._stderr_fd = None + if not selector.get_map() and self.poll() is None: + time.sleep(0.01) + self.wait(timeout=0) + finally: + selector.close() + self._stdout_fd = None + self._stderr_fd = None + return ( + bytes(output) if had_stdout else None, + bytes(errors) if had_stderr else None, + ) + + +def _linux_child_file_descriptors( + mode: Any, + target_fd: int, +) -> tuple[int | None, int | None]: + if mode is None: + return None, None + if mode == subprocess.PIPE: + read_fd, write_fd = os.pipe() + if target_fd == 0: + return write_fd, read_fd + return read_fd, write_fd + if mode == subprocess.DEVNULL: + flags = os.O_RDONLY if target_fd == 0 else os.O_WRONLY + descriptor = os.open(os.devnull, flags) + return None, descriptor + if mode == subprocess.STDOUT and target_fd == 2: + return None, subprocess.STDOUT + raise TraceError("Linux fork/exec containment received unsupported stream controls") + + +def _start_linux_blocked_process(command: list[str], launch: dict[str, Any]) -> _LinuxForkExecProcess: + controls = dict(launch) + unknown_controls = set(controls) - { + "cwd", "env", "executable", "pass_fds", "stderr", "stdin", "stdout"} + if unknown_controls: + raise TraceError( + f"Linux fork/exec containment received unsupported controls: {sorted(unknown_controls)}") + executable = str(controls.pop("executable", command[0])) + pass_fds = tuple(int(fd) for fd in controls.pop("pass_fds", ())) + environment = controls.pop("env", None) + working_directory = controls.pop("cwd", None) + opened_descriptors = set() + try: + stdin_parent, stdin_child = _linux_child_file_descriptors(controls.pop("stdin", None), 0) + opened_descriptors.update( + descriptor for descriptor in (stdin_parent, stdin_child) + if descriptor is not None and descriptor != subprocess.STDOUT) + stdout_parent, stdout_child = _linux_child_file_descriptors(controls.pop("stdout", None), 1) + opened_descriptors.update( + descriptor for descriptor in (stdout_parent, stdout_child) + if descriptor is not None and descriptor != subprocess.STDOUT) + stderr_parent, stderr_child = _linux_child_file_descriptors(controls.pop("stderr", None), 2) + opened_descriptors.update( + descriptor for descriptor in (stderr_parent, stderr_child) + if descriptor is not None and descriptor != subprocess.STDOUT) + barrier_read, barrier_write = os.pipe() + opened_descriptors.update((barrier_read, barrier_write)) + error_read, error_write = os.pipe() + opened_descriptors.update((error_read, error_write)) + os.set_inheritable(error_write, False) + child_descriptors = [ + descriptor for descriptor in (stdin_child, stdout_child, stderr_child) + if descriptor is not None and descriptor != subprocess.STDOUT] + if any(descriptor <= 2 for descriptor in child_descriptors): + raise TraceError("Linux fork/exec containment requires intact standard descriptors") + except BaseException: + for descriptor in opened_descriptors: + try: + os.close(descriptor) + except OSError: + pass + raise + try: + process_id = os.fork() + except BaseException: + for descriptor in opened_descriptors: + os.close(descriptor) + raise + if process_id == 0: + try: + os.close(barrier_write) + os.close(error_read) + for parent_fd in (stdin_parent, stdout_parent, stderr_parent): + if parent_fd is not None: + os.close(parent_fd) + for child_fd, target_fd in ( + (stdin_child, 0), (stdout_child, 1), (stderr_child, 2)): + if child_fd == subprocess.STDOUT: + os.dup2(1, 2) + elif child_fd is not None: + os.dup2(child_fd, target_fd) + for descriptor in pass_fds: + os.set_inheritable(descriptor, True) + keep = {0, 1, 2, barrier_read, error_write, *pass_fds} + for descriptor_name in os.listdir("/proc/self/fd"): + descriptor = int(descriptor_name) + if descriptor not in keep: + try: + os.close(descriptor) + except OSError: + pass + if os.read(barrier_read, 1) != b"1": + os._exit(126) + os.close(barrier_read) + if working_directory is not None: + os.chdir(working_directory) + os.execve(executable, command, os.environ if environment is None else environment) + except BaseException as error: + try: + os.write(error_write, f"{type(error).__name__}: {error}".encode("ascii", "backslashreplace")) + finally: + os._exit(127) + process = _LinuxForkExecProcess( + command, + process_id, + barrier_fd=barrier_write, + exec_error_fd=error_read, + stdin_fd=stdin_parent, + stdout_fd=stdout_parent, + stderr_fd=stderr_parent, + ) + parent_close_failures = [] + for descriptor, component in ( + (barrier_read, "linux-parent-barrier-read-close"), + (error_write, "linux-parent-error-write-close"), + (stdin_child, "linux-parent-stdin-child-close"), + (stdout_child, "linux-parent-stdout-child-close"), + (stderr_child, "linux-parent-stderr-child-close")): + if descriptor is None or descriptor == subprocess.STDOUT: + continue + try: + os.close(descriptor) + except BaseException as error: + parent_close_failures.append(_IntegrityFailure(component, error)) + if parent_close_failures: + primary_failure = parent_close_failures.pop(0) + cleanup = process.abort_blocked() + parent_close_failures.extend(cleanup.failures) + parent_close_failures.extend(process.close_streams()) + raise ExecutionIntegrityError( + f"Linux blocked-child setup primary failure " + f"[{type(primary_failure.error).__name__}: {primary_failure.error}]; " + f"secondary integrity failures: {_format_integrity_failures(parent_close_failures)}", + primary_error=primary_failure.error, + secondary_errors=parent_close_failures, + quiescence_proven=False, + ) from primary_failure.error + return process + + def _linux_direct_children() -> set[int]: children = set() task_root = Path("/proc/self/task") @@ -932,17 +1254,28 @@ def _linux_task_ids() -> set[int]: raise TraceError(f"cannot read Linux subreaper task identities: {error}") from error -def _linux_enable_child_subreaper() -> None: +def _linux_get_child_subreaper() -> int: libc = ctypes.CDLL(None, use_errno=True) - if libc.prctl(LINUX_PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) != 0: - error_number = ctypes.get_errno() - raise TraceError(f"cannot enable Linux child subreaper: {os.strerror(error_number)}") enabled = ctypes.c_int() if libc.prctl(LINUX_PR_GET_CHILD_SUBREAPER, ctypes.byref(enabled), 0, 0, 0) != 0: error_number = ctypes.get_errno() - raise TraceError(f"cannot verify Linux child subreaper: {os.strerror(error_number)}") - if enabled.value != 1: - raise TraceError("Linux child subreaper did not remain enabled") + raise TraceError(f"cannot read Linux child subreaper: {os.strerror(error_number)}") + if enabled.value not in (0, 1): + raise TraceError(f"invalid Linux child subreaper state {enabled.value}") + return enabled.value + + +def _linux_set_child_subreaper(value: int) -> None: + if value not in (0, 1): + raise TraceError(f"invalid Linux child subreaper state {value}") + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(LINUX_PR_SET_CHILD_SUBREAPER, value, 0, 0, 0) != 0: + error_number = ctypes.get_errno() + raise TraceError(f"cannot set Linux child subreaper: {os.strerror(error_number)}") + actual = _linux_get_child_subreaper() + if actual != value: + raise TraceError( + f"Linux child subreaper verification failed: expected {value}, got {actual}") def _linux_open_pidfd(process_id: int) -> int: @@ -1016,36 +1349,73 @@ def _linux_signal_owned_children( def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + global _LINUX_SUBREAPER_POISONED if not _LINUX_SUBREAPER_LOCK.acquire(blocking=False): raise TraceError("Linux subreaper containment is already active") process = None pidfd = None + prior_subreaper = None + exec_released = False try: - _linux_enable_child_subreaper() + if _LINUX_SUBREAPER_POISONED: + raise TraceError("Linux subreaper containment supervisor is not reusable") + prior_subreaper = _linux_get_child_subreaper() + if prior_subreaper != 1: + _linux_set_child_subreaper(1) _linux_require_pidfd_support() if len(_linux_task_ids()) != 1: raise TraceError("Linux subreaper containment requires a single-threaded supervisor") if _linux_direct_children(): raise TraceError("Linux subreaper containment process owns unrelated children") - process = subprocess.Popen(command, **launch) + process = _start_linux_blocked_process(command, launch) pidfd = _linux_open_pidfd(process.pid) - return _ProcessContainment( + containment = _ProcessContainment( process=process, linux_root_pidfd=pidfd, linux_lock_held=True, + linux_prior_subreaper=prior_subreaper, ) + exec_released = True + containment.linux_exec_released = True + process.release_exec() + return containment except BaseException as primary_error: failures = [] + startup_quiescence_proven = False + if isinstance(primary_error, ExecutionIntegrityError): + failures.extend(primary_error.secondary_errors) + startup_quiescence_proven = primary_error.quiescence_proven + primary_error = primary_error.primary_error or primary_error if process is not None: failed_containment = _ProcessContainment( process=process, linux_root_pidfd=pidfd, linux_lock_held=True, + linux_prior_subreaper=prior_subreaper, + linux_exec_released=exec_released, + ) + if exec_released: + cleanup = _terminate_process_tree(failed_containment) + failures.extend(cleanup.failures) + startup_quiescence_proven = cleanup.quiescence_proven + else: + cleanup = process.abort_blocked() + failures.extend(cleanup.failures) + startup_quiescence_proven = cleanup.quiescence_proven + close_failures = _close_process_containment( + failed_containment, + quiescence_proven=startup_quiescence_proven, ) - cleanup = _terminate_process_tree(failed_containment) - failures.extend(cleanup.failures) - failures.extend(_close_process_containment(failed_containment)) + failures.extend(close_failures) + startup_quiescence_proven = ( + pidfd is not None and startup_quiescence_proven and not close_failures) else: + if prior_subreaper is not None: + try: + _linux_set_child_subreaper(prior_subreaper) + except BaseException as error: + _LINUX_SUBREAPER_POISONED = True + failures.append(_IntegrityFailure("linux-subreaper-restore", error)) _LINUX_SUBREAPER_LOCK.release() if failures: raise ExecutionIntegrityError( @@ -1054,7 +1424,7 @@ def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) - f"secondary integrity failures: {_format_integrity_failures(failures)}", primary_error=primary_error, secondary_errors=failures, - quiescence_proven=False, + quiescence_proven=startup_quiescence_proven, ) from primary_error if process is not None: raise ExecutionIntegrityError( @@ -1062,7 +1432,7 @@ def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) - f"[{type(primary_error).__name__}: {primary_error}]", primary_error=primary_error, secondary_errors=[], - quiescence_proven=False, + quiescence_proven=startup_quiescence_proven, ) from primary_error raise @@ -1116,8 +1486,10 @@ def _close_windows_process_handle(process: subprocess.Popen[bytes]) -> None: process_handle = getattr(process, "_handle", None) if process_handle is None: return - if not _windows_kernel32().CloseHandle(int(process_handle)): - raise _windows_error("cannot close process handle") + close = getattr(process_handle, "Close", None) + if not callable(close): + raise TraceError("subprocess process handle does not expose owned Close()") + close() process._handle = None @@ -1267,7 +1639,12 @@ def _run_contained_process( None, error, failures, containment, cleanup.quiescence_proven, True) -def _close_process_containment(containment: _ProcessContainment | None) -> list[_IntegrityFailure]: +def _close_process_containment( + containment: _ProcessContainment | None, + *, + quiescence_proven: bool, +) -> list[_IntegrityFailure]: + global _LINUX_SUBREAPER_POISONED if containment is None: return [] failures = [] @@ -1288,7 +1665,25 @@ def _close_process_containment(containment: _ProcessContainment | None) -> list[ containment.linux_root_pidfd = None except BaseException as error: failures.append(_IntegrityFailure("linux-root-pidfd-close", error)) + if isinstance(containment.process, _LinuxForkExecProcess): + failures.extend(containment.process.close_streams()) if containment.linux_lock_held: + if not quiescence_proven: + failures.append(_IntegrityFailure( + "linux-subreaper-restore", + TraceError("Linux child-subreaper state cannot be restored before full tree quiescence"))) + _LINUX_SUBREAPER_POISONED = True + elif containment.linux_prior_subreaper is None: + failures.append(_IntegrityFailure( + "linux-subreaper-restore", + TraceError("Linux child-subreaper prior state is missing"))) + _LINUX_SUBREAPER_POISONED = True + else: + try: + _linux_set_child_subreaper(containment.linux_prior_subreaper) + except BaseException as error: + _LINUX_SUBREAPER_POISONED = True + failures.append(_IntegrityFailure("linux-subreaper-restore", error)) containment.linux_lock_held = False _LINUX_SUBREAPER_LOCK.release() return failures @@ -1517,7 +1912,13 @@ def run_approved_executable( os.close(descriptor) except BaseException as error: integrity_failures.append(_IntegrityFailure("executable-descriptor-close", error)) - integrity_failures.extend(_close_process_containment(containment)) + containment_failures = _close_process_containment( + containment, + quiescence_proven=quiescence_proven, + ) + integrity_failures.extend(containment_failures) + if containment_failures: + quiescence_proven = False if result is not None and primary_error is None: try: stdout = _decode_subprocess_stream( From 76a685c3f6f36f01ede5e82fb1c35810c602d435 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 16:37:13 -0700 Subject: [PATCH 42/56] trace : contain Linux execution in native namespace helper Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 699 ++++++++++++----- tools/deepseek-v41-trace/CMakeLists.txt | 47 +- tools/deepseek-v41-trace/README.md | 6 +- .../generate-containment-helper-receipt.py | 44 ++ .../linux-containment-helper.cpp | 535 +++++++++++++ tools/deepseek-v41-trace/run_llama.py | 6 +- tools/deepseek-v41-trace/run_matrix.py | 15 +- tools/deepseek-v41-trace/test-install.cmake | 37 +- tools/deepseek-v41-trace/trace_format.py | 740 ++++++++++-------- 9 files changed, 1609 insertions(+), 520 deletions(-) create mode 100644 tools/deepseek-v41-trace/generate-containment-helper-receipt.py create mode 100644 tools/deepseek-v41-trace/linux-containment-helper.cpp diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index db123248176d..0654b486562a 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -497,6 +497,16 @@ def replace_watchdog_events(root: Path, phase: str, events: list[dict[str, objec replace_audit_record(root, phase, "watchdog", record) +def fixture_containment_helper(revision: str, digest: str = "d" * 64) -> dict[str, object]: + return { + "format": "dsv41-containment-helper", + "version": 1, + "revision": revision, + "filename": "llama-deepseek-v41-containment-helper", + "sha256": digest, + } + + def fixture_prompt_builder_policy( prompt: bytes, *, @@ -526,6 +536,7 @@ def fixture_prompt_builder_policy( "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, "executable_path": builder_path, "executable_sha256": builder_sha256, + "containment_helper": fixture_containment_helper("a" * 40), "source_root": source_root, "runtime_receipt": { "format": "dsv41-runtime-receipt", @@ -572,6 +583,12 @@ def materialize_policy_runtime(policy: dict[str, object]) -> None: component["sha256"] = trace.sha256_file(path) path.chmod(0o555) executable = Path(policy["executable_path"]) + containment_helper = policy.get("containment_helper") + if containment_helper is not None: + helper = Path(policy["install_root"]) / "bin" / containment_helper["filename"] + helper.write_bytes(b"containment-helper") + helper.chmod(0o555) + containment_helper["sha256"] = trace.sha256_file(helper) if executable.exists(): executable.chmod(0o555) library_root.chmod(0o555) @@ -603,6 +620,10 @@ def fixture_install_trust(policy: dict[str, object]) -> dict[str, object]: for component in policy["runtime_receipt"]["components"] }, } + containment_helper = policy.get("containment_helper") + if containment_helper is not None: + paths[str(install_root / "bin" / containment_helper["filename"])] = ( + containment_helper["sha256"]) directory_paths = set() for path in paths: current = Path(path).parent @@ -1005,6 +1026,7 @@ def manifest( "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, "executable_path": result["candidate"]["executable_path"], "executable_sha256": result["candidate"]["executable_sha256"], + "containment_helper": fixture_containment_helper(result["candidate"]["revision"]), "runtime_profile": copy.deepcopy(result["build"]["runtime_profile"]), "runtime_receipt": runtime_receipt, } @@ -1398,6 +1420,8 @@ def _verifier_for_runtime( "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, "executable_path": manifest_record["candidate"]["executable_path"], "executable_sha256": manifest_record["candidate"]["executable_sha256"], + "containment_helper": fixture_containment_helper( + manifest_record["candidate"]["revision"]), "runtime_profile": copy.deepcopy(manifest_record["build"]["runtime_profile"]), "runtime_receipt": receipt, } @@ -1976,16 +2000,22 @@ def test_approved_executable_uses_linux_descriptor_path(self) -> None: with tempfile.TemporaryDirectory() as temp: install = Path(temp).resolve() / "install" executable = install / "bin" / "approved" + helper = install / "bin" / "llama-deepseek-v41-containment-helper" library = install / "lib" / "libapproved.so" executable.parent.mkdir(parents=True) library.parent.mkdir() executable.write_bytes(b"approved") + helper.write_bytes(b"helper") library.write_bytes(b"approved library") executable.chmod(0o555) + helper.chmod(0o555) library.chmod(0o555) policy = { + "revision": "a" * 40, "install_root": str(install), "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "containment_helper": fixture_containment_helper( + "a" * 40, trace.sha256_file(helper)), "runtime_receipt": { "components": [{ "component": "approved", @@ -2378,143 +2408,303 @@ def test_approved_executable_timeout_contains_setsid_descendant(self) -> None: time.sleep(1.2) self.assertFalse(marker.exists()) - def test_linux_subreaper_signals_stable_pidfds_not_numeric_ids(self) -> None: - process = mock.Mock(pid=77) + def test_linux_native_helper_signals_only_stable_pidfds(self) -> None: containment = trace._ProcessContainment( - process=process, linux_root_pidfd=90, linux_lock_held=True) - with mock.patch.object( - trace, "_linux_direct_children", return_value={77, 78}), mock.patch.object( - trace, "_linux_open_pidfd", return_value=91) as open_pidfd, mock.patch.object( - trace, "_linux_signal_pidfd") as signal_pidfd, mock.patch.object( - trace.os, "close") as close, mock.patch.object(trace.os, "kill") as numeric_kill: + process=mock.Mock(pid=77), + linux_root_pidfd=90, + linux_namespace_pidfd=91, + linux_lock_held=True, + linux_exec_released=True, + ) + with mock.patch.object(trace, "_linux_signal_pidfd") as signal_pidfd, mock.patch.object( + trace.os, "kill") as numeric_kill: failures = trace._linux_signal_owned_children(containment, trace.signal.SIGKILL) self.assertEqual(failures, []) - open_pidfd.assert_called_once_with(78) self.assertEqual( signal_pidfd.call_args_list, - [mock.call(90, trace.signal.SIGKILL), mock.call(91, trace.signal.SIGKILL)], + [mock.call(91, trace.signal.SIGKILL), mock.call(90, trace.signal.SIGKILL)], ) - close.assert_called_once_with(91) numeric_kill.assert_not_called() - def test_linux_subreaper_rejects_unrelated_child_ownership(self) -> None: + def test_linux_native_helper_boundary_precedes_target_release(self) -> None: + start_source = inspect.getsource(trace._start_linux_native_helper) + spawn_source = inspect.getsource(trace._start_linux_native_helper_process) + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + self.assertNotIn("os.fork", start_source + spawn_source) + self.assertIn("os.posix_spawn", spawn_source) + self.assertLess(spawn_source.index("os.posix_spawn"), spawn_source.index("_linux_open_pidfd")) + self.assertLess(start_source.index("process.root_pidfd"), start_source.index("process.release_exec")) + self.assertIn("PR_SET_PDEATHSIG", helper_source) + for flag in ("CLONE_NEWUSER", "CLONE_NEWPID", "CLONE_NEWNS", "CLONE_PIDFD"): + self.assertIn(flag, helper_source) + self.assertLess( + helper_source.index("namespace_owner owned_namespace"), + helper_source.index('"uid_map"'), + ) + self.assertIn('"uid_map"', helper_source) + self.assertIn('mount("proc", "/proc", "proc"', helper_source) + self.assertLess( + helper_source.index("target PID namespace did not bind helper lifetime"), + helper_source.index("send_pidfd(config.protocol_fd"), + ) + self.assertLess( + helper_source.index("send_pidfd(config.protocol_fd"), + helper_source.index('!= \"EXEC\"'), + ) + + def test_posix_spawn_does_not_run_registered_atfork_callback(self) -> None: + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp) / "atfork-ran" + probe = ( + "import os,sys\n" + "marker=sys.argv[1]\n" + "os.register_at_fork(after_in_child=lambda: open(marker,'w',encoding='ascii').write('ran'))\n" + "pid=os.posix_spawn('/usr/bin/true',['/usr/bin/true'],os.environ)\n" + "os.waitpid(pid,0)\n" + "raise SystemExit(1 if os.path.exists(marker) else 0)\n" + ) + subprocess.run( + [sys.executable, "-c", probe, str(marker)], + check=True, + env={"PATH": os.environ.get("PATH", "")}, + ) + self.assertFalse(marker.exists()) + + def test_linux_native_helper_inherited_signal_defenses_are_explicit(self) -> None: + spawn_source = inspect.getsource(trace._start_linux_native_helper_process) + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + self.assertIn("setsigmask=_all_catchable_signals()", spawn_source) + self.assertIn("setsigdef=_all_catchable_signals()", spawn_source) + self.assertIn("require_initial_signal_state();", helper_source) + self.assertLess( + helper_source.index("require_initial_signal_state();"), + helper_source.index('send_packet(config.protocol_fd, \"READY\")'), + ) + + def test_linux_native_helper_without_pidfd_support_fails_before_spawn(self) -> None: lock = mock.Mock() lock.acquire.return_value = True - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( - trace, "_linux_set_child_subreaper") as set_subreaper, mock.patch.object( - trace, "_linux_require_pidfd_support"), mock.patch.object( - trace, "_linux_task_ids", return_value={1}), mock.patch.object( - trace, "_linux_direct_children", return_value={42}), mock.patch.object( - trace, "_start_linux_blocked_process") as start, self.assertRaisesRegex( - trace.TraceError, "owns unrelated children"): - trace._start_linux_subreaper_process(["approved"], {}) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_linux_require_pidfd_support", + side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( + trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( + trace.TraceError, "pidfd unavailable"): + trace._start_linux_native_helper(["approved"], {}) start.assert_not_called() - self.assertEqual(set_subreaper.call_args_list, [mock.call(1), mock.call(0)]) lock.release.assert_called_once() - def test_linux_subreaper_boundary_precedes_target_execution(self) -> None: - source = inspect.getsource(trace._start_linux_subreaper_process) - self.assertLess(source.index("_linux_get_child_subreaper"), source.index("_start_linux_blocked_process")) - self.assertLess(source.index("_linux_task_ids"), source.index("_start_linux_blocked_process")) - self.assertLess(source.index("_linux_direct_children"), source.index("_start_linux_blocked_process")) - self.assertLess(source.index("_start_linux_blocked_process"), source.index("_linux_open_pidfd")) - self.assertLess(source.index("_linux_open_pidfd"), source.index("process.release_exec")) - self.assertNotIn("killpg", source) - self.assertNotIn("os.kill(", inspect.getsource(trace._linux_signal_owned_children)) - - def test_linux_subreaper_without_pidfd_support_fails_before_launch(self) -> None: + def test_linux_helper_pidfd_failure_aborts_before_protocol_release(self) -> None: lock = mock.Mock() lock.acquire.return_value = True - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( - trace, "_linux_set_child_subreaper") as set_subreaper, mock.patch.object( - trace, "_linux_require_pidfd_support", side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( - trace, "_start_linux_blocked_process") as start, self.assertRaisesRegex( - trace.TraceError, "pidfd unavailable"): - trace._start_linux_subreaper_process(["approved"], {}) - start.assert_not_called() - self.assertEqual(set_subreaper.call_args_list, [mock.call(1), mock.call(0)]) + primary = OSError("pidfd failed") + launch_error = trace.ExecutionIntegrityError( + "helper pidfd failed", + primary_error=primary, + secondary_errors=[], + quiescence_proven=False, + ) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_start_linux_native_helper_process", side_effect=launch_error), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper(["approved"], {}) + self.assertFalse(raised.exception.quiescence_proven) + self.assertIs(raised.exception.primary_error, primary) lock.release.assert_called_once() - def test_linux_prelaunch_primary_and_restore_failure_are_both_retained(self) -> None: + def test_linux_helper_pidfd_open_failure_reaps_blocked_helper(self) -> None: + parent_socket = mock.Mock() + child_socket = mock.Mock() + parent_socket.fileno.return_value = 50 + child_socket.fileno.return_value = 51 + primary = OSError("pidfd failed") lock = mock.Mock() lock.acquire.return_value = True - primary = trace.TraceError("pidfd unavailable") - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( - trace, "_linux_set_child_subreaper", side_effect=[None, OSError("restore failed")]), mock.patch.object( - trace, "_linux_require_pidfd_support", side_effect=primary), self.assertRaises( + with mock.patch.object( + trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( + trace.os, "get_inheritable", return_value=False), mock.patch.object( + trace.os, "set_inheritable"), mock.patch.object( + trace.os, "posix_spawn", return_value=71), mock.patch.object( + trace, "_linux_open_pidfd", side_effect=primary), mock.patch.object( + trace.os, "waitpid", return_value=(71, 0)), mock.patch.object( + trace.os, "kill") as numeric_kill, self.assertRaises( trace.ExecutionIntegrityError) as raised: - trace._start_linux_subreaper_process(["approved"], {}) + trace._start_linux_native_helper( + ["/bin/true"], + { + "_containment_helper_path": "/approved/helper", + "_containment_helper_descriptor": 40, + }, + ) self.assertIs(raised.exception.primary_error, primary) - self.assertIs(raised.exception.__cause__, primary) self.assertFalse(raised.exception.quiescence_proven) + parent_socket.shutdown.assert_called_once_with(trace.socket.SHUT_RDWR) + parent_socket.close.assert_called_once() + child_socket.close.assert_called_once() + numeric_kill.assert_not_called() + + def test_linux_post_spawn_cleanup_preserves_all_failures_and_reaps(self) -> None: + parent_socket = mock.Mock() + child_socket = mock.Mock() + parent_socket.fileno.return_value = 50 + child_socket.fileno.return_value = 51 + child_socket.close.side_effect = OSError("child protocol close failed") + restore_error = OSError("inheritability restore failed") + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object( + trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( + trace.os, "get_inheritable", return_value=False), mock.patch.object( + trace.os, "set_inheritable", side_effect=[None, restore_error]), mock.patch.object( + trace.os, "posix_spawn", return_value=71), mock.patch.object( + trace, "_linux_open_pidfd", return_value=90), mock.patch.object( + trace, "_linux_signal_pidfd") as signal_pidfd, mock.patch.object( + trace.os, "waitpid", return_value=(71, 0)), mock.patch.object( + trace.os, "close"), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper( + ["/bin/true"], + { + "_containment_helper_path": "/approved/helper", + "_containment_helper_descriptor": 40, + }, + ) + self.assertIs(raised.exception.primary_error, restore_error) self.assertEqual( [failure.component for failure in raised.exception.secondary_errors], - ["linux-subreaper-restore"], + ["linux-helper-child-protocol-close"], ) - lock.release.assert_called_once() + self.assertFalse(raised.exception.quiescence_proven) + signal_pidfd.assert_not_called() + parent_socket.close.assert_called_once() - def test_linux_pidfd_failure_aborts_blocked_child_before_exec(self) -> None: + def test_linux_failed_startup_reap_retains_locked_authority(self) -> None: lock = mock.Mock() lock.acquire.return_value = True - process = mock.Mock(pid=71) - process.abort_blocked.return_value = trace._ContainmentCleanup([], True) - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_get_child_subreaper", return_value=0), mock.patch.object( - trace, "_linux_set_child_subreaper") as set_subreaper, mock.patch.object( + primary = OSError("launch cleanup failed") + reap_failure = trace._IntegrityFailure( + "linux-blocked-child-reap", subprocess.TimeoutExpired(["helper"], 5)) + process = mock.Mock( + pid=71, + root_pidfd=90, + namespace_pidfd=None, + launch_primary_error=primary, + launch_integrity_failures=[], + ) + process.abort_blocked.return_value = trace._ContainmentCleanup( + [reap_failure], False) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace, "_LINUX_POISONED_CONTAINMENT", None), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( - trace, "_linux_direct_children", return_value=set()), mock.patch.object( - trace, "_start_linux_blocked_process", return_value=process), mock.patch.object( - trace, "_linux_open_pidfd", side_effect=OSError("pidfd failed")), self.assertRaises( - trace.ExecutionIntegrityError) as raised: - trace._start_linux_subreaper_process(["approved"], {}) + trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( + trace.os, "close") as close: + with self.assertRaises(trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper(["approved"], {}) + poisoned = trace._LINUX_HELPER_POISONED + poisoned_containment = trace._LINUX_POISONED_CONTAINMENT + self.assertIs(raised.exception.primary_error, primary) self.assertFalse(raised.exception.quiescence_proven) - process.abort_blocked.assert_called_once() - process.release_exec.assert_not_called() - self.assertEqual(set_subreaper.call_args_list, [mock.call(1), mock.call(0)]) - lock.release.assert_called_once() + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-blocked-child-reap", "linux-helper-teardown"], + ) + self.assertTrue(poisoned) + self.assertIs(poisoned_containment.process, process) + process.close_streams.assert_not_called() + close.assert_not_called() + lock.release.assert_not_called() + + def test_linux_blocked_helper_reap_retries_after_protocol_close(self) -> None: + protocol = mock.Mock() + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + process, + "wait", + side_effect=[subprocess.TimeoutExpired(["approved"], 5), 0], + ) as wait: + cleanup = process.abort_blocked() + self.assertFalse(cleanup.quiescence_proven) + self.assertEqual( + [failure.component for failure in cleanup.failures], + ["linux-blocked-child-reap"], + ) + self.assertEqual(wait.call_count, 2) + protocol.shutdown.assert_called_once_with(trace.socket.SHUT_RDWR) + protocol.close.assert_called_once() + self.assertIsNone(process._protocol_socket) + + def test_linux_namespace_unavailable_never_sends_exec(self) -> None: + protocol = mock.Mock() + protocol.recvmsg.side_effect = [ + (b"READY", [], 0, None), + (b"PREPARED", [], 0, None), + ] + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with self.assertRaisesRegex(trace.TraceError, "did not provide one namespace pidfd"): + process.release_exec() + self.assertEqual(protocol.sendall.call_args_list, [mock.call(b"PREPARE")]) + self.assertIsNone(process.namespace_pidfd) - def test_linux_exec_barrier_releases_only_after_pidfd(self) -> None: + def test_linux_namespace_authority_precedes_release_completion(self) -> None: events = [] lock = mock.Mock() lock.acquire.return_value = True - process = mock.Mock(pid=71) - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_get_child_subreaper", return_value=1), mock.patch.object( + process = mock.Mock( + pid=71, + root_pidfd=90, + namespace_pidfd=91, + launch_primary_error=None, + launch_integrity_failures=[], + ) + process.release_exec.side_effect = lambda: events.append("protocol") + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( - trace, "_linux_direct_children", return_value=set()), mock.patch.object( - trace, "_start_linux_blocked_process", - side_effect=lambda *_args: events.append("blocked") or process), mock.patch.object( - trace, "_linux_open_pidfd", - side_effect=lambda pid: events.append(f"pidfd:{pid}") or 90), mock.patch.object( - trace.os, "close"), mock.patch.object( - trace, "_linux_set_child_subreaper") as restore: - process.release_exec.side_effect = lambda: events.append("exec") - containment = trace._start_linux_subreaper_process(["approved"], {}) - self.assertEqual(containment.linux_root_pidfd, 90) - self.assertEqual(containment.linux_prior_subreaper, 1) - self.assertTrue(containment.linux_exec_released) - self.assertEqual(trace._close_process_containment( - containment, quiescence_proven=True), []) - self.assertEqual(events, ["blocked", "pidfd:71", "exec"]) - restore.assert_called_once_with(1) - lock.release.assert_called_once() - - def test_linux_blocked_child_never_signals_after_identity_loss(self) -> None: - process = trace._LinuxForkExecProcess( + trace, "_start_linux_native_helper_process", + side_effect=lambda *_args: events.append("spawn") or process), mock.patch.object( + trace, "_linux_pidfd_has_exited", return_value=False): + containment = trace._start_linux_native_helper(["approved"], {}) + self.assertEqual(events, ["spawn", "protocol"]) + self.assertEqual(containment.linux_root_pidfd, 90) + self.assertEqual(containment.linux_namespace_pidfd, 91) + self.assertTrue(containment.linux_exec_released) + + def test_linux_native_helper_process_identity_loss_never_signals_numeric_pid(self) -> None: + process = trace._LinuxNativeHelperProcess( ["approved"], 71, - barrier_fd=80, - exec_error_fd=81, + protocol_socket=mock.Mock(), stdin_fd=None, stdout_fd=None, stderr_fd=None, @@ -2525,76 +2715,152 @@ def test_linux_blocked_child_never_signals_after_identity_loss(self) -> None: process.kill() kill.assert_not_called() - @unittest.skipUnless(sys.platform == "linux", "Linux fork/exec containment test") - def test_linux_fork_exec_barrier_blocks_target_until_release(self) -> None: + def test_linux_native_helper_source_binds_supervisor_death_and_namespace_lifecycle(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + self.assertGreaterEqual(helper_source.count("PR_SET_PDEATHSIG"), 2) + self.assertIn("CLONE_NEWPID", helper_source) + self.assertIn("kill(-1, SIGKILL)", helper_source) + self.assertIn("getppid() != expected_parent", helper_source) + self.assertIn("getppid() != 0", helper_source) + + @unittest.skipUnless( + sys.platform == "linux" and os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER"), + "native Linux containment helper was not executed on this host", + ) + def test_linux_native_helper_parent_death_boundary(self) -> None: + import array + import socket + + helper = Path(os.environ["DSV41_NATIVE_CONTAINMENT_HELPER"]).resolve(strict=True) with tempfile.TemporaryDirectory() as temp: - marker = Path(temp) / "target-ran" - process = trace._start_linux_blocked_process( - ["/bin/sh", "-c", "printf ran > \"$1\"", "sh", str(marker)], - {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE}, + root = Path(temp) + socket_path = root / "pidfds.sock" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + listener.bind(str(socket_path)) + listener.listen(2) + listener.settimeout(10) + target_source = ( + "import array,os,socket,sys,time\n" + "def report(label):\n" + " s=socket.socket(socket.AF_UNIX,socket.SOCK_SEQPACKET)\n" + " s.connect(sys.argv[1])\n" + " fd=os.pidfd_open(os.getpid())\n" + " rights=array.array('i',[fd])\n" + " s.sendmsg([label.encode('ascii')],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,rights)])\n" + " os.close(fd);s.close()\n" + "report('target')\n" + "if os.fork()==0:\n" + " os.setsid();report('descendant')\n" + " while True: time.sleep(1)\n" + "while True: time.sleep(1)\n" ) - self.assertFalse(marker.exists()) - process.release_exec() - stdout, stderr = process.communicate(timeout=5) - self.assertEqual(process.returncode, 0) - self.assertEqual(stdout, b"") - self.assertEqual(stderr, b"") - self.assertEqual(marker.read_text(encoding="ascii"), "ran") - self.assertEqual(process.close_streams(), []) - - @unittest.skipUnless(sys.platform == "linux", "Linux fork/exec containment test") - def test_linux_blocked_child_abort_reaps_without_target_execution(self) -> None: - with tempfile.TemporaryDirectory() as temp: - marker = Path(temp) / "target-ran" - before = len(os.listdir("/proc/self/fd")) - process = trace._start_linux_blocked_process( - ["/bin/sh", "-c", "printf ran > \"$1\"", "sh", str(marker)], - {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE}, + supervisor_source = ( + "import os,sys,time\n" + "from pathlib import Path\n" + "import trace_format as trace\n" + "helper=Path(sys.argv[1])\n" + "target=os.open(sys.executable,os.O_RDONLY)\n" + "helper_fd=os.open(helper,os.O_RDONLY)\n" + "launch={'executable':f'/proc/self/fd/{target}'," + "'pass_fds':(target,)," + "'_containment_helper_path':f'/proc/self/fd/{helper_fd}'," + "'_containment_helper_descriptor':helper_fd," + "'stdout':-3,'stderr':-3}\n" + "containment=trace._start_linux_native_helper(" + "[sys.executable,'-c',sys.argv[2],sys.argv[3]],launch)\n" + "while True: time.sleep(1)\n" ) - cleanup = process.abort_blocked() - self.assertTrue(cleanup.quiescence_proven) - self.assertEqual(cleanup.failures, []) - self.assertFalse(marker.exists()) - self.assertEqual(process.close_streams(), []) - self.assertEqual(len(os.listdir("/proc/self/fd")), before) - - def test_linux_process_stream_close_reports_every_failure(self) -> None: - process = trace._LinuxForkExecProcess( + environment = { + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(Path(__file__).parents[1] / "tools/deepseek-v41-trace"), + } + supervisor = subprocess.Popen( + [sys.executable, "-c", supervisor_source, str(helper), target_source, str(socket_path)], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + pidfds = [] + try: + labels = set() + while len(pidfds) < 2: + try: + connection, _address = listener.accept() + except TimeoutError: + if supervisor.poll() is not None: + stderr = supervisor.stderr.read() + self.skipTest(f"native PID namespace unavailable: {stderr.strip()}") + raise + with connection: + descriptors = array.array("i") + data, ancillary, flags, _address = connection.recvmsg( + 32, socket.CMSG_SPACE(descriptors.itemsize)) + self.assertEqual(flags, 0) + for level, kind, content in ancillary: + if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: + descriptors.frombytes(content[:descriptors.itemsize]) + self.assertEqual(len(descriptors), 1) + labels.add(data.decode("ascii")) + pidfds.append(descriptors[0]) + self.assertEqual(labels, {"target", "descendant"}) + supervisor.kill() + supervisor.wait(timeout=5) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and not all( + trace._linux_pidfd_has_exited(pidfd) for pidfd in pidfds): + time.sleep(0.02) + self.assertTrue(all( + trace._linux_pidfd_has_exited(pidfd) for pidfd in pidfds)) + finally: + for pidfd in pidfds: + if not trace._linux_pidfd_has_exited(pidfd): + trace._linux_signal_pidfd(pidfd, trace.signal.SIGKILL) + os.close(pidfd) + if supervisor.poll() is None: + supervisor.kill() + supervisor.wait(timeout=5) + listener.close() + + def test_linux_native_helper_stream_close_reports_every_failure(self) -> None: + protocol = mock.Mock() + protocol.close.side_effect = OSError("protocol") + process = trace._LinuxNativeHelperProcess( ["approved"], 71, - barrier_fd=80, - exec_error_fd=81, + protocol_socket=protocol, stdin_fd=82, stdout_fd=83, stderr_fd=84, ) with mock.patch.object(trace.os, "close", side_effect=[ - OSError("barrier"), None, OSError("stdin"), None, OSError("stderr")]): + OSError("stdin"), None, OSError("stderr")]): failures = process.close_streams() self.assertEqual( [failure.component for failure in failures], [ - "linux-process-fd-close:barrier", + "linux-process-fd-close:protocol", "linux-process-fd-close:stdin", "linux-process-fd-close:stderr", ], ) - self.assertEqual( - [str(failure.error) for failure in failures], - ["barrier", "stdin", "stderr"], - ) - def test_linux_fork_exec_rejects_controls_without_descriptor_leaks(self) -> None: - for launch in ( - {"shell": True}, - {"close_fds": False}, - {"stdin": subprocess.PIPE, "stderr": object()}, - ): - with self.subTest(controls=sorted(launch)): - before = len(os.listdir("/dev/fd")) - with self.assertRaisesRegex(trace.TraceError, "unsupported"): - trace._start_linux_blocked_process(["/bin/true"], launch) - self.assertEqual(len(os.listdir("/dev/fd")), before) + def test_linux_native_helper_rejects_controls_before_spawn(self) -> None: + for launch in ({"shell": True}, {"close_fds": False}, {"cwd": "/tmp"}): + with self.subTest(controls=sorted(launch)), mock.patch.object( + trace.os, "posix_spawn") as spawn, self.assertRaises(trace.TraceError): + trace._start_linux_native_helper_process( + ["/bin/true"], + { + "_containment_helper_path": "/approved/helper", + "_containment_helper_descriptor": 40, + **launch, + }, + ) + spawn.assert_not_called() def test_unproven_posix_containment_fails_closed_before_setsid_escape(self) -> None: if sys.platform == "linux": @@ -2739,40 +3005,7 @@ def Close(self) -> None: self.assertIsNone(process._handle) kernel32.assert_not_called() - def test_linux_subreaper_restores_exact_prior_state(self) -> None: - for prior_state in (0, 1): - with self.subTest(prior_state=prior_state): - lock = mock.Mock() - containment = trace._ProcessContainment( - process=mock.Mock(), - linux_lock_held=True, - linux_prior_subreaper=prior_state, - ) - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_linux_set_child_subreaper") as restore: - failures = trace._close_process_containment( - containment, quiescence_proven=True) - self.assertEqual(failures, []) - restore.assert_called_once_with(prior_state) - lock.release.assert_called_once() - - def test_linux_subreaper_restore_failure_poisoning_blocks_completion(self) -> None: - lock = mock.Mock() - containment = trace._ProcessContainment( - process=mock.Mock(), - linux_lock_held=True, - linux_prior_subreaper=0, - ) - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_set_child_subreaper", side_effect=OSError("restore failed")): - failures = trace._close_process_containment( - containment, quiescence_proven=True) - self.assertTrue(trace._LINUX_SUBREAPER_POISONED) - self.assertEqual([failure.component for failure in failures], ["linux-subreaper-restore"]) - lock.release.assert_called_once() - - def test_linux_subreaper_restores_after_timeout_and_target_exception(self) -> None: + def test_linux_native_helper_closes_after_timeout_and_target_exception(self) -> None: for primary in ( subprocess.TimeoutExpired(["approved"], 1), OSError("target failed"), @@ -2784,17 +3017,17 @@ def test_linux_subreaper_restores_after_timeout_and_target_exception(self) -> No containment = trace._ProcessContainment( process=process, linux_root_pidfd=90, + linux_namespace_pidfd=91, linux_lock_held=True, - linux_prior_subreaper=0, linux_exec_released=True, ) cleanup = trace._ContainmentCleanup([], True) with mock.patch.object( trace, "_start_contained_process", return_value=containment), mock.patch.object( trace, "_terminate_process_tree", return_value=cleanup), mock.patch.object( - trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace.os, "close"), mock.patch.object( - trace, "_linux_set_child_subreaper") as restore: + trace, "_linux_pidfd_has_exited", return_value=True): result = trace._run_contained_process( ["approved"], label="approved executable", @@ -2808,39 +3041,90 @@ def test_linux_subreaper_restores_after_timeout_and_target_exception(self) -> No ) self.assertIs(result.primary_error, primary) self.assertEqual(close_failures, []) - restore.assert_called_once_with(0) lock.release.assert_called_once() - def test_linux_subreaper_does_not_restore_without_tree_quiescence(self) -> None: + def test_linux_native_helper_teardown_requires_tree_quiescence(self) -> None: lock = mock.Mock() containment = trace._ProcessContainment( process=mock.Mock(), linux_lock_held=True, - linux_prior_subreaper=0, ) - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", False), mock.patch.object( - trace, "_linux_set_child_subreaper") as restore: + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace, "_LINUX_POISONED_CONTAINMENT", None): failures = trace._close_process_containment( containment, quiescence_proven=False, ) - self.assertTrue(trace._LINUX_SUBREAPER_POISONED) - restore.assert_not_called() - self.assertEqual([failure.component for failure in failures], ["linux-subreaper-restore"]) - lock.release.assert_called_once() + self.assertTrue(trace._LINUX_HELPER_POISONED) + self.assertIs(trace._LINUX_POISONED_CONTAINMENT, containment) + self.assertEqual([failure.component for failure in failures], ["linux-helper-teardown"]) + lock.release.assert_not_called() + + def test_linux_native_helper_rejects_concurrent_reuse(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = False + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( + trace.TraceError, "already active"): + trace._start_linux_native_helper(["approved"], {}) + start.assert_not_called() + lock.release.assert_not_called() - def test_linux_poisoned_subreaper_supervisor_rejects_reuse(self) -> None: + def test_linux_poisoned_helper_supervisor_rejects_reuse(self) -> None: lock = mock.Mock() lock.acquire.return_value = True - with mock.patch.object(trace, "_LINUX_SUBREAPER_LOCK", lock), mock.patch.object( - trace, "_LINUX_SUBREAPER_POISONED", True), mock.patch.object( - trace, "_start_linux_blocked_process") as start, self.assertRaisesRegex( + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", True), mock.patch.object( + trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( trace.TraceError, "not reusable"): - trace._start_linux_subreaper_process(["approved"], {}) + trace._start_linux_native_helper(["approved"], {}) start.assert_not_called() lock.release.assert_called_once() + def test_linux_missing_helper_completion_blocks_containment_gate(self) -> None: + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + process.returncode = 0 + containment = trace._ProcessContainment( + process=process, + linux_root_pidfd=90, + linux_namespace_pidfd=91, + linux_lock_held=True, + linux_exec_released=True, + ) + with mock.patch.object( + process, "communicate", return_value=(b"", b"")), mock.patch.object( + trace, "_linux_signal_owned_children", return_value=[]), mock.patch.object( + trace, "_linux_pidfd_has_exited", return_value=True): + cleanup = trace._terminate_process_tree(containment) + self.assertFalse(cleanup.quiescence_proven) + self.assertEqual( + [failure.component for failure in cleanup.failures], + ["linux-helper-completion"], + ) + + def test_containment_helper_policy_mutations_fail_closed(self) -> None: + for field, value in ( + ("revision", "b" * 40), + ("filename", "other-helper"), + ("sha256", "not-a-digest"), + ("version", 2)): + with self.subTest(field=field): + policy = fixture_prompt_builder_policy(b"prompt") + policy["containment_helper"][field] = value + with self.assertRaisesRegex(trace.TraceError, "containment helper receipt"): + trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + def test_containment_handle_close_failure_forces_quiescence_false(self) -> None: with tempfile.TemporaryDirectory() as temp: install = Path(temp).resolve() / "install" @@ -4016,9 +4300,18 @@ def test_ds4_explicit_quiescence_false_always_blocks_post_attestation(self) -> N "windows-process-reap", "windows-process-termination", "windows-job-assignment", - "linux-child-ownership", + "linux-helper-completion", + "linux-helper-protocol-shutdown", + "linux-helper-teardown", "linux-root-pidfd-close", - "linux-subreaper-restore", + "linux-namespace-pidfd-close", + "containment-helper-descriptor-close", + "executable-descriptor-close", + "runtime-descriptor-close", + "linux-process-fd-close:protocol", + "linux-process-fd-close:stdin", + "linux-process-fd-close:stdout", + "linux-process-fd-close:stderr", "windows-process-handle-close", "windows-thread-handle-close", "containment-handle-close"): diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt index c47133812cc2..202c008e16fb 100644 --- a/tools/deepseek-v41-trace/CMakeLists.txt +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -2,7 +2,7 @@ set(TARGET llama-deepseek-v41-trace) set(DSV41_INSTALL_COMPONENT DeepSeekV41Trace) if(NOT BUILD_SHARED_LIBS) - message(STATUS "Skipping DeepSeek V4.1 trace exporter because BUILD_SHARED_LIBS is disabled") + message(STATUS "Skipping DeepSeek V4.1 trace tools because BUILD_SHARED_LIBS is disabled") return() endif() @@ -30,6 +30,30 @@ foreach(backend IN LISTS DSV41_BACKEND_TARGETS) endforeach() list(REMOVE_DUPLICATES DSV41_RECEIPT_TARGETS) +set(CONTAINMENT_HELPER_TARGET llama-deepseek-v41-containment-helper) +add_executable(${CONTAINMENT_HELPER_TARGET} linux-containment-helper.cpp) +target_compile_features(${CONTAINMENT_HELPER_TARGET} PRIVATE cxx_std_17) +target_compile_definitions( + ${CONTAINMENT_HELPER_TARGET} + PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") +set(DSV41_CONTAINMENT_HELPER_RECEIPT + "${CMAKE_CURRENT_BINARY_DIR}/dsv41-containment-helper-receipt.json") +add_custom_command( + OUTPUT "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + COMMAND + ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/generate-containment-helper-receipt.py" + --output "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + --helper "$" + --revision "${DSV41_BUILD_REVISION}" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/generate-containment-helper-receipt.py" + ${CONTAINMENT_HELPER_TARGET} + VERBATIM) +add_custom_target( + dsv41-containment-helper-receipt + DEPENDS "${DSV41_CONTAINMENT_HELPER_RECEIPT}") + if(WIN32) set(DSV41_RUNTIME_PROFILE co-located) else() @@ -102,7 +126,7 @@ set_source_files_properties( PROPERTIES OBJECT_DEPENDS "${DSV41_RECEIPT_HEADER}") add_executable(${TARGET} llama-trace.cpp) -add_dependencies(${TARGET} dsv41-runtime-receipt) +add_dependencies(${TARGET} dsv41-runtime-receipt dsv41-containment-helper-receipt) target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) target_compile_features(${TARGET} PRIVATE cxx_std_17) @@ -117,7 +141,7 @@ endif() set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) add_executable(${PROMPT_TARGET} prompt-builder.cpp) -add_dependencies(${PROMPT_TARGET} dsv41-runtime-receipt) +add_dependencies(${PROMPT_TARGET} dsv41-runtime-receipt dsv41-containment-helper-receipt) target_link_libraries(${PROMPT_TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) target_include_directories(${PROMPT_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_compile_features(${PROMPT_TARGET} PRIVATE cxx_std_17) @@ -131,6 +155,16 @@ if(NOT WIN32) endif() if(LLAMA_TOOLS_INSTALL) + install( + TARGETS ${CONTAINMENT_HELPER_TARGET} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install( + FILES "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + DESTINATION "share/deepseek-v41-trace" + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ GROUP_READ WORLD_READ) install( TARGETS ${TARGET} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -155,7 +189,10 @@ endif() if(LLAMA_BUILD_TESTS) set(MANIFEST_TEST_TARGET test-deepseek41-trace-manifest) add_executable(${MANIFEST_TEST_TARGET} llama-trace.cpp) - add_dependencies(${MANIFEST_TEST_TARGET} dsv41-runtime-receipt) + add_dependencies( + ${MANIFEST_TEST_TARGET} + dsv41-runtime-receipt + dsv41-containment-helper-receipt) target_link_libraries( ${MANIFEST_TEST_TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) @@ -206,6 +243,8 @@ if(LLAMA_BUILD_TESTS) -DDSV41_CONFIG=$ -DDSV41_PYTHON=${Python3_EXECUTABLE} -DDSV41_RECEIPT=${DSV41_RECEIPT_JSON} + -DDSV41_CONTAINMENT_HELPER=$ + -DDSV41_CONTAINMENT_HELPER_RECEIPT=${DSV41_CONTAINMENT_HELPER_RECEIPT} -DDSV41_VERIFY_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/verify-runtime-install.py -P ${CMAKE_CURRENT_SOURCE_DIR}/test-install.cmake) set_property(TEST test-deepseek41-trace-install PROPERTY LABELS main) diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 1f4a4bb751b5..09aa9ebf8c91 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -17,9 +17,9 @@ Signer trust is external to the bundle. `APPROVED_TRACE_SIGNERS` maps one restri Executable trust is also external to the bundle and to the reviewed source revision. `APPROVED_EXECUTABLE_APPROVERS`, `APPROVED_CANDIDATE_EXPORTERS`, and `APPROVED_PROMPT_BUILDERS` are intentionally empty in production source. An authorized run must receive a canonical detached approval policy plus its OpenSSH signature and an externally expected approver principal. Both files must be absolute canonical non-symlinked one-link regular files outside every protected output root. Their trusted root and owner come only from `APPROVED_EXECUTABLE_APPROVERS`; the fixed `ssh-keygen` verifies namespace `dsv41-executable-approval-v2` before either executable can run. This avoids an impossible same-revision self-reference: the policy records the artifact producer revision and exact executable hashes, while a separate `verifier_revision` records the harness revision that consumes the policy. -The candidate exporter approval binds the exact producer revision, base revision, binary diff SHA-256, canonical install root and exporter path, exporter SHA-256, runtime profile, and complete embedded runtime receipt. The prompt-builder approval binds its producer revision, canonical install/source roots and executable path, executable SHA-256, exact runtime profile and receipt, model and corpus identities, complete tokenizer policy, and exact prompt hash, byte count, context, and decode configuration for every authorized case. The tokenizer policy explicitly binds `add_bos`, `parse_special`, `detokenize_special`, leading-BOS removal, and exact token round-trip. The builder invocation, native result, signed provenance, execution authorization, candidate environment, and candidate manifest must all agree with it. +The candidate exporter approval binds the exact producer revision, base revision, binary diff SHA-256, canonical install root and exporter path, exporter SHA-256, runtime profile, complete embedded runtime receipt, and the exact revision, filename, and SHA-256 of the native Linux containment helper. The prompt-builder approval binds the same helper identity with its producer revision, canonical install/source roots and executable path, executable SHA-256, exact runtime profile and receipt, model and corpus identities, complete tokenizer policy, and exact prompt hash, byte count, context, and decode configuration for every authorized case. The tokenizer policy explicitly binds `add_bos`, `parse_special`, `detokenize_special`, leading-BOS removal, and exact token round-trip. The builder invocation, native result, signed provenance, execution authorization, candidate environment, and candidate manifest must all agree with it. -Every approved install tree is verified before launch. The install owner is externally selected and must differ from the unprivileged execution identity. Every ancestor is canonical, non-symlinked, trusted-owned, ACL-free, and not writable by the execution identity or group/other. Every approved executable and receipt library is a canonical, ACL-free, non-writable, one-link regular file with the exact owner and SHA-256. On Linux, both the exporter and prompt builder are opened without following the final symlink and executed through the retained `/proc/self/fd` descriptor. Receipt-library descriptors remain open through process completion, while the immutable canonical install hierarchy prevents pathname substitution during loader consumption. Both native tools report the actual canonical loaded project-library paths and hashes; the prompt provenance and candidate manifest require the loaded closure to equal the approved receipt before and after protected work. Ordinary user-owned install smoke verifies layout, RPATH, and bytes only; it does not establish production trusted-root authorization. +Every approved install tree is verified before launch. The install owner is externally selected and must differ from the unprivileged execution identity. Every ancestor is canonical, non-symlinked, trusted-owned, ACL-free, and not writable by the execution identity or group/other. Every approved executable, containment helper, and receipt library is a canonical, ACL-free, non-writable, one-link regular file with the exact owner and SHA-256. On Linux, the exporter, prompt builder, and containment helper are opened without following the final symlink and launched through retained `/proc/self/fd` descriptors. Receipt-library descriptors remain open through process completion, while the immutable canonical install hierarchy prevents pathname substitution during loader consumption. Both native tools report the actual canonical loaded project-library paths and hashes; the prompt provenance and candidate manifest require the loaded closure to equal the approved receipt before and after protected work. Ordinary user-owned install smoke verifies layout, RPATH, helper receipt, version, and bytes only; it does not establish production trusted-root authorization. The signed execution authorization records the complete approval-policy SHA-256, verifier revision, tokenizer-policy SHA-256, and both selected approval record IDs, policy hashes, and install-trust hashes. Candidate-derived attestations and prompt provenance are evidence only and must exactly equal those external records. Seal creation performs the same semantic manifest, provenance, event, and coverage validation as standalone verification before invoking the signer. @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded no-unrelated-child subreaper owner and stable pidfds. The launcher snapshots the exact prior child-subreaper state, forks a trusted bootstrap that cannot execute target code, acquires the root pidfd while that child remains behind a one-shot exec barrier, and only then releases the target. A failed pidfd acquisition kills and reaps the exact blocked child before the barrier can release. After execution, the owner kills the exact root, drains every reparented descendant, reaps all owned children, proves its child set empty without numeric PID or process-group signaling, restores and verifies the exact prior subreaper state, and closes every containment handle before releasing the ownership lock. A restoration failure poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, termination, assignment, reaping, empty-set proof, subreaper restoration, or pidfd, Job, process, or thread handle teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the unprivileged execution identity, makes mount propagation private, removes the inherited host procfs view, and mounts a procfs view owned by the new PID namespace. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. The namespace init binds its lifetime to the helper, executes the target only after release, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, clone3, pidfds, parent-death binding, mapping, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/generate-containment-helper-receipt.py b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py new file mode 100644 index 000000000000..073380a6d2d7 --- /dev/null +++ b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--helper", required=True, type=Path) + parser.add_argument("--revision", required=True) + args = parser.parse_args() + if re.fullmatch(r"[0-9a-f]{40}", args.revision) is None: + parser.error("revision must be an exact full Git revision") + helper = args.helper.resolve(strict=True) + receipt = { + "format": "dsv41-containment-helper", + "version": 1, + "revision": args.revision, + "filename": helper.name, + "sha256": sha256_file(helper), + } + content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_suffix(args.output.suffix + ".tmp") + temporary.write_bytes(content) + os.replace(temporary, args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp new file mode 100644 index 000000000000..20d6ade1aa5e --- /dev/null +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -0,0 +1,535 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(SYS_clone3) +#define SYS_clone3 435 +#endif +#if !defined(SYS_pidfd_send_signal) +#define SYS_pidfd_send_signal 424 +#endif +#if !defined(SYS_pidfd_open) +#define SYS_pidfd_open 434 +#endif + +extern char ** environ; +#endif + +#ifndef DSV41_BUILD_REVISION +#define DSV41_BUILD_REVISION "unknown" +#endif + +namespace { + +struct options { + int protocol_fd = -1; + int expected_parent = -1; + std::string exec_path; + std::vector keep_fds; + std::vector target_argv; +}; + +int parse_positive_int(const char * value, const char * label) { + char * end = nullptr; + errno = 0; + const long parsed = std::strtol(value, &end, 10); + if (errno != 0 || end == value || *end != '\0' || parsed <= 0 || + parsed > std::numeric_limits::max()) { + throw std::runtime_error(std::string("invalid ") + label); + } + return static_cast(parsed); +} + +options parse_options(int argc, char ** argv) { + options result; + int index = 1; + while (index < argc && std::strcmp(argv[index], "--") != 0) { + if (std::strcmp(argv[index], "--protocol-fd") == 0 && index + 1 < argc) { + result.protocol_fd = parse_positive_int(argv[index + 1], "protocol descriptor"); + index += 2; + } else if (std::strcmp(argv[index], "--expected-parent") == 0 && index + 1 < argc) { + result.expected_parent = parse_positive_int(argv[index + 1], "expected parent"); + index += 2; + } else if (std::strcmp(argv[index], "--exec-path") == 0 && index + 1 < argc) { + result.exec_path = argv[index + 1]; + index += 2; + } else if (std::strcmp(argv[index], "--keep-fd") == 0 && index + 1 < argc) { + result.keep_fds.push_back(parse_positive_int(argv[index + 1], "retained descriptor")); + index += 2; + } else { + throw std::runtime_error("unknown containment helper argument"); + } + } + if (index >= argc || std::strcmp(argv[index], "--") != 0) { + throw std::runtime_error("containment helper target separator is missing"); + } + ++index; + if (result.protocol_fd < 0 || result.expected_parent < 0 || result.exec_path.empty() || + index >= argc) { + throw std::runtime_error("containment helper arguments are incomplete"); + } + for (; index < argc; ++index) { + result.target_argv.push_back(argv[index]); + } + result.target_argv.push_back(nullptr); + return result; +} + +#if defined(__linux__) + +bool retained_fd(int fd, const std::vector & keep_fds, int extra_fd) { + if (fd >= 0 && fd <= STDERR_FILENO) { + return true; + } + if (fd == extra_fd) { + return true; + } + for (int keep_fd : keep_fds) { + if (fd == keep_fd) { + return true; + } + } + return false; +} + +void close_checked(int fd, const char * label) { + if (close(fd) != 0) { + throw std::runtime_error(std::string("cannot close ") + label); + } +} + +void close_unneeded_fds(const std::vector & keep_fds, int extra_fd) { + DIR * directory = opendir("/proc/self/fd"); + if (directory == nullptr) { + throw std::runtime_error("cannot open procfs descriptor directory"); + } + const int directory_fd = dirfd(directory); + while (dirent * entry = readdir(directory)) { + char * end = nullptr; + const long parsed = std::strtol(entry->d_name, &end, 10); + if (end == entry->d_name || *end != '\0' || parsed < 0 || + parsed > std::numeric_limits::max()) { + continue; + } + const int fd = static_cast(parsed); + if (fd != directory_fd && !retained_fd(fd, keep_fds, extra_fd)) { + close_checked(fd, "unneeded descriptor"); + } + } + if (closedir(directory) != 0) { + throw std::runtime_error("cannot close procfs descriptor directory"); + } +} + +void require_initial_signal_state() { + sigset_t mask; + if (sigprocmask(SIG_SETMASK, nullptr, &mask) != 0) { + throw std::runtime_error("cannot read containment helper signal mask"); + } + for (int signal_number = 1; signal_number < NSIG; ++signal_number) { + if (signal_number == SIGKILL || signal_number == SIGSTOP) { + continue; + } + struct sigaction action {}; + if (sigaction(signal_number, nullptr, &action) != 0) { + if (errno == EINVAL) { + continue; + } + throw std::runtime_error("cannot read containment helper signal disposition"); + } + if (sigismember(&mask, signal_number) != 1) { + throw std::runtime_error("containment helper inherited an unblocked signal"); + } + if (action.sa_handler != SIG_DFL) { + throw std::runtime_error("containment helper inherited a signal handler"); + } + } +} + +void unblock_default_signals() { + sigset_t empty; + sigemptyset(&empty); + if (sigprocmask(SIG_SETMASK, &empty, nullptr) != 0) { + throw std::runtime_error("cannot unblock containment helper signals"); + } +} + +void set_parent_death(pid_t expected_parent) { + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { + throw std::runtime_error("cannot set containment parent-death signal"); + } + if (getppid() != expected_parent) { + throw std::runtime_error("containment parent changed before lifecycle binding"); + } +} + +void write_all(int fd, const void * data, size_t size) { + const char * cursor = static_cast(data); + while (size > 0) { + const ssize_t written = write(fd, cursor, size); + if (written < 0) { + if (errno == EINTR) { + continue; + } + throw std::runtime_error("containment protocol write failed"); + } + cursor += written; + size -= static_cast(written); + } +} + +void write_mapping_file(int process_directory, const char * name, const std::string & content) { + const int fd = openat(process_directory, name, O_WRONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) { + throw std::runtime_error(std::string("cannot open namespace ") + name); + } + write_all(fd, content.data(), content.size()); + close_checked(fd, "namespace mapping descriptor"); +} + +std::string receive_packet(int fd) { + char buffer[128]; + const ssize_t size = recv(fd, buffer, sizeof(buffer), 0); + if (size <= 0) { + throw std::runtime_error("containment protocol closed"); + } + return std::string(buffer, static_cast(size)); +} + +void send_packet(int fd, const char * packet) { + const size_t size = std::strlen(packet); + if (send(fd, packet, size, MSG_NOSIGNAL) != static_cast(size)) { + throw std::runtime_error("containment protocol send failed"); + } +} + +void send_pidfd(int fd, int pidfd) { + char payload[] = "PREPARED"; + iovec vector {payload, sizeof(payload) - 1}; + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int))] {}; + msghdr message {}; + message.msg_iov = &vector; + message.msg_iovlen = 1; + message.msg_control = control; + message.msg_controllen = sizeof(control); + cmsghdr * header = CMSG_FIRSTHDR(&message); + header->cmsg_level = SOL_SOCKET; + header->cmsg_type = SCM_RIGHTS; + header->cmsg_len = CMSG_LEN(sizeof(int)); + std::memcpy(CMSG_DATA(header), &pidfd, sizeof(pidfd)); + if (sendmsg(fd, &message, MSG_NOSIGNAL) != static_cast(sizeof(payload) - 1)) { + throw std::runtime_error("cannot send namespace pidfd"); + } +} + +bool pidfd_has_exited(int pidfd) { + pollfd descriptor {pidfd, POLLIN, 0}; + const int result = poll(&descriptor, 1, 0); + if (result < 0) { + throw std::runtime_error("cannot query containment pidfd"); + } + return result > 0 && (descriptor.revents & (POLLIN | POLLHUP | POLLERR)) != 0; +} + +int wait_status_exit_code(int status) { + if (WIFEXITED(status)) { + return WEXITSTATUS(status); + } + if (WIFSIGNALED(status)) { + return 128 + WTERMSIG(status); + } + return 125; +} + +class namespace_owner { +public: + namespace_owner(pid_t pid, int pidfd) : pid_(pid), pidfd_(pidfd) { + } + + ~namespace_owner() { + if (reaped_) { + return; + } + if (pidfd_ >= 0) { + syscall(SYS_pidfd_send_signal, pidfd_, SIGKILL, nullptr, 0); + } + while (waitpid(pid_, nullptr, 0) < 0 && errno == EINTR) { + } + if (pidfd_ >= 0) { + close(pidfd_); + } + } + + int pidfd() const { + return pidfd_; + } + + int wait() { + int status = 0; + while (waitpid(pid_, &status, 0) < 0) { + if (errno != EINTR) { + throw std::runtime_error("cannot reap target PID namespace"); + } + } + reaped_ = true; + return status; + } + + void close_pidfd() { + const int descriptor = pidfd_; + pidfd_ = -1; + if (close(descriptor) != 0) { + throw std::runtime_error("cannot close namespace pidfd"); + } + } + +private: + pid_t pid_; + int pidfd_; + bool reaped_ = false; +}; + +[[noreturn]] void run_namespace_init( + int release_fd, + int ready_fd, + int mapping_fd, + int protocol_fd, + int helper_pidfd, + const options & config) { + try { + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { + _exit(125); + } + if (getppid() != 0 || pidfd_has_exited(helper_pidfd)) { + _exit(125); + } + close_checked(helper_pidfd, "namespace helper pidfd"); + close_checked(protocol_fd, "namespace protocol descriptor"); + write_all(ready_fd, "B", 1); + char mapped = 0; + ssize_t mapped_size; + do { + mapped_size = read(mapping_fd, &mapped, 1); + } while (mapped_size < 0 && errno == EINTR); + if (mapped_size != 1 || mapped != 'M') { + _exit(125); + } + close_checked(mapping_fd, "namespace mapping descriptor"); + if (setresgid(0, 0, 0) != 0 || setresuid(0, 0, 0) != 0) { + _exit(125); + } + if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0 || + umount2("/proc", MNT_DETACH) != 0 || + mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr) != 0) { + _exit(125); + } + write_all(ready_fd, "R", 1); + close_checked(ready_fd, "namespace readiness descriptor"); + char release = 0; + ssize_t received; + do { + received = read(release_fd, &release, 1); + } while (received < 0 && errno == EINTR); + if (received != 1 || release != 'X') { + _exit(125); + } + close_checked(release_fd, "namespace release descriptor"); + const pid_t target_pid = fork(); + if (target_pid < 0) { + _exit(125); + } + if (target_pid == 0) { + for (int signal_number = 1; signal_number < NSIG; ++signal_number) { + if (signal_number == SIGKILL || signal_number == SIGSTOP) { + continue; + } + struct sigaction action {}; + action.sa_handler = SIG_DFL; + sigemptyset(&action.sa_mask); + sigaction(signal_number, &action, nullptr); + } + sigset_t empty; + sigemptyset(&empty); + sigprocmask(SIG_SETMASK, &empty, nullptr); + close_unneeded_fds(config.keep_fds, -1); + execve(config.exec_path.c_str(), config.target_argv.data(), environ); + _exit(127); + } + int target_status = 0; + while (waitpid(target_pid, &target_status, 0) < 0) { + if (errno != EINTR) { + _exit(125); + } + } + kill(-1, SIGKILL); + while (waitpid(-1, nullptr, 0) >= 0 || errno == EINTR) { + } + _exit(wait_status_exit_code(target_status)); + } catch (...) { + _exit(125); + } +} + +int run_linux_helper(options config) { + require_initial_signal_state(); + set_parent_death(config.expected_parent); + close_unneeded_fds(config.keep_fds, config.protocol_fd); + unblock_default_signals(); + send_packet(config.protocol_fd, "READY"); + if (receive_packet(config.protocol_fd) != "PREPARE") { + throw std::runtime_error("containment PREPARE packet is invalid"); + } + + int release_pipe[2] {-1, -1}; + if (pipe2(release_pipe, O_CLOEXEC) != 0) { + throw std::runtime_error("cannot create namespace release pipe"); + } + int ready_pipe[2] {-1, -1}; + if (pipe2(ready_pipe, O_CLOEXEC) != 0) { + close(release_pipe[0]); + close(release_pipe[1]); + throw std::runtime_error("cannot create namespace readiness pipe"); + } + int mapping_pipe[2] {-1, -1}; + if (pipe2(mapping_pipe, O_CLOEXEC) != 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + throw std::runtime_error("cannot create namespace mapping pipe"); + } + const int helper_pidfd = static_cast(syscall(SYS_pidfd_open, getpid(), 0)); + if (helper_pidfd < 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + close(mapping_pipe[0]); + close(mapping_pipe[1]); + throw std::runtime_error("cannot open stable helper identity"); + } + int namespace_pidfd = -1; + clone_args arguments {}; + arguments.flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD; + arguments.pidfd = reinterpret_cast(&namespace_pidfd); + arguments.exit_signal = SIGCHLD; + const pid_t namespace_init = static_cast( + syscall(SYS_clone3, &arguments, sizeof(arguments))); + if (namespace_init < 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + close(mapping_pipe[0]); + close(mapping_pipe[1]); + close(helper_pidfd); + throw std::runtime_error(std::string("cannot create target PID namespace: ") + std::strerror(errno)); + } + if (namespace_init == 0) { + close_checked(release_pipe[1], "namespace release writer"); + close_checked(ready_pipe[0], "namespace readiness reader"); + close_checked(mapping_pipe[1], "namespace mapping writer"); + run_namespace_init( + release_pipe[0], ready_pipe[1], mapping_pipe[0], + config.protocol_fd, helper_pidfd, config); + } + namespace_owner owned_namespace(namespace_init, namespace_pidfd); + close_checked(release_pipe[0], "helper release reader"); + close_checked(ready_pipe[1], "helper readiness writer"); + close_checked(mapping_pipe[0], "helper mapping reader"); + char ready = 0; + ssize_t ready_size; + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + if (ready_size != 1 || ready != 'B') { + throw std::runtime_error("target PID namespace did not bind helper lifetime"); + } + const std::string process_path = "/proc/" + std::to_string(namespace_init); + const int process_directory = open( + process_path.c_str(), O_PATH | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); + if (process_directory < 0 || pidfd_has_exited(namespace_pidfd)) { + if (process_directory >= 0) { + close_checked(process_directory, "namespace process directory"); + } + throw std::runtime_error("cannot retain target namespace mapping identity"); + } + write_mapping_file(process_directory, "setgroups", "deny\n"); + write_mapping_file( + process_directory, "uid_map", + "0 " + std::to_string(getuid()) + " 1\n"); + write_mapping_file( + process_directory, "gid_map", + "0 " + std::to_string(getgid()) + " 1\n"); + close_checked(process_directory, "namespace process directory"); + write_all(mapping_pipe[1], "M", 1); + close_checked(mapping_pipe[1], "helper mapping writer"); + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + close_checked(ready_pipe[0], "helper readiness reader"); + close_checked(helper_pidfd, "helper self pidfd"); + if (ready_size != 1 || ready != 'R') { + throw std::runtime_error("target PID namespace setup did not complete"); + } + send_pidfd(config.protocol_fd, owned_namespace.pidfd()); + if (receive_packet(config.protocol_fd) != "EXEC") { + throw std::runtime_error("containment EXEC packet is invalid"); + } + write_all(release_pipe[1], "X", 1); + close_checked(release_pipe[1], "helper release writer"); + send_packet(config.protocol_fd, "RELEASED"); + const int status = owned_namespace.wait(); + owned_namespace.close_pidfd(); + send_packet(config.protocol_fd, "COMPLETE"); + return wait_status_exit_code(status); +} + +#endif + +} + +int main(int argc, char ** argv) { + try { + if (argc == 2 && std::strcmp(argv[1], "--version") == 0) { + std::cout << "deepseek-v41-containment-helper " << DSV41_BUILD_REVISION << '\n'; + return 0; + } +#if defined(__linux__) + return run_linux_helper(parse_options(argc, argv)); +#else + (void) argc; + (void) argv; + std::cerr << "Linux PID namespace containment is unavailable on this platform\n"; + return 125; +#endif + } catch (const std::exception & error) { + std::cerr << error.what() << '\n'; + return 125; + } +} diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 2cc8e4193c07..24da2a85d829 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -40,6 +40,7 @@ TraceError, TraceVerifier, approval_binding, + approved_containment_helper_identity, approved_executable_identity, approved_runtime_file_identities, bind_execution_authorization, @@ -545,7 +546,10 @@ def main() -> int: ) runtime_identities = approved_runtime_file_identities( candidate_policy, label="candidate exporter") - candidate_trust = install_trust_evidence(exporter_identity, runtime_identities) + helper_identity = approved_containment_helper_identity( + candidate_policy, label="candidate exporter") + candidate_trust = install_trust_evidence( + exporter_identity, runtime_identities, (helper_identity,)) exporter_sha256 = exporter_identity.sha256 if args.candidate_revision != candidate_policy["revision"] or ( args.base_revision != candidate_policy["base_revision"]) or ( diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index ac590179f811..cc7a1994ed26 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -22,6 +22,7 @@ REQUIRED_EXPERT_SLOTS, TraceError, approval_binding, + approved_containment_helper_identity, approved_executable_identity, approved_prompt_record, approved_runtime_file_identities, @@ -130,6 +131,8 @@ def prepare_prompt( ) runtime_identities = approved_runtime_file_identities( builder_policy, label="prompt builder") + helper_identity = approved_containment_helper_identity( + builder_policy, label="prompt builder") expected_source = Path(builder_policy["source_root"]) / "tests" / "corpus" / corpus_name if source_corpus != expected_source or sha256_file(source_corpus) != corpus_sha256 or ( sha256_file(corpus) != corpus_sha256): @@ -200,7 +203,8 @@ def prepare_prompt( if prompt_sha256 != expected_prompt["prompt_sha256"] or ( prompt_byte_count != expected_prompt["prompt_byte_count"]): raise RuntimeError("prompt builder output differs from external approval") - trust_evidence = install_trust_evidence(builder_identity, runtime_identities) + trust_evidence = install_trust_evidence( + builder_identity, runtime_identities, (helper_identity,)) record = { "format": "dsv41-prompt-provenance", "version": 1, @@ -313,8 +317,10 @@ def main() -> int: ) candidate_runtime_identities = approved_runtime_file_identities( candidate_policy, label="candidate exporter") + candidate_helper_identity = approved_containment_helper_identity( + candidate_policy, label="candidate exporter") candidate_trust = install_trust_evidence( - candidate_identity, candidate_runtime_identities) + candidate_identity, candidate_runtime_identities, (candidate_helper_identity,)) prompt_builder = args.llama_prompt_builder prompt_identity = approved_executable_identity( prompt_builder, @@ -326,7 +332,10 @@ def main() -> int: ) prompt_runtime_identities = approved_runtime_file_identities( prompt_policy, label="prompt builder") - prompt_trust = install_trust_evidence(prompt_identity, prompt_runtime_identities) + prompt_helper_identity = approved_containment_helper_identity( + prompt_policy, label="prompt builder") + prompt_trust = install_trust_evidence( + prompt_identity, prompt_runtime_identities, (prompt_helper_identity,)) if args.candidate_revision != candidate_policy["revision"] or ( args.base_revision != candidate_policy["base_revision"]) or ( args.candidate_diff_sha256 != candidate_policy["diff_sha256"]): diff --git a/tools/deepseek-v41-trace/test-install.cmake b/tools/deepseek-v41-trace/test-install.cmake index d07ff43b2f93..57588eae2db9 100644 --- a/tools/deepseek-v41-trace/test-install.cmake +++ b/tools/deepseek-v41-trace/test-install.cmake @@ -2,7 +2,8 @@ if(NOT DEFINED DSV41_BUILD_DIR OR NOT DEFINED DSV41_INSTALL_ROOT OR NOT DEFINED DSV41_COMPONENT OR NOT DEFINED DSV41_EXECUTABLE OR NOT DEFINED DSV41_SOURCE_EXECUTABLE OR NOT DEFINED DSV41_REVISION OR NOT DEFINED DSV41_PYTHON OR - NOT DEFINED DSV41_RECEIPT OR NOT DEFINED DSV41_VERIFY_SCRIPT) + NOT DEFINED DSV41_RECEIPT OR NOT DEFINED DSV41_CONTAINMENT_HELPER OR + NOT DEFINED DSV41_CONTAINMENT_HELPER_RECEIPT OR NOT DEFINED DSV41_VERIFY_SCRIPT) message(FATAL_ERROR "DeepSeek V4.1 install smoke test arguments are incomplete") endif() @@ -25,6 +26,24 @@ if(NOT source_executable_sha256 STREQUAL installed_executable_sha256) message(FATAL_ERROR "installed DeepSeek V4.1 trace executable differs from linked bytes") endif() +get_filename_component(containment_helper_name "${DSV41_CONTAINMENT_HELPER}" NAME) +file(SHA256 "${DSV41_CONTAINMENT_HELPER}" source_helper_sha256) +file(SHA256 "${DSV41_INSTALL_ROOT}/bin/${containment_helper_name}" installed_helper_sha256) +if(NOT source_helper_sha256 STREQUAL installed_helper_sha256) + message(FATAL_ERROR "installed DeepSeek V4.1 containment helper differs from linked bytes") +endif() +file( + SHA256 "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + source_containment_helper_receipt_sha256) +file( + SHA256 + "${DSV41_INSTALL_ROOT}/share/deepseek-v41-trace/dsv41-containment-helper-receipt.json" + installed_containment_helper_receipt_sha256) +if(NOT source_containment_helper_receipt_sha256 STREQUAL + installed_containment_helper_receipt_sha256) + message(FATAL_ERROR "installed DeepSeek V4.1 containment helper receipt differs from build bytes") +endif() + execute_process( COMMAND "${DSV41_PYTHON}" "${DSV41_VERIFY_SCRIPT}" --receipt "${DSV41_RECEIPT}" @@ -62,3 +81,19 @@ endif() if(NOT smoke_output MATCHES "commit ${DSV41_REVISION}") message(FATAL_ERROR "installed DeepSeek V4.1 trace reported the wrong revision:\n${smoke_output}") endif() + +execute_process( + COMMAND "${DSV41_INSTALL_ROOT}/bin/${containment_helper_name}" --version + RESULT_VARIABLE helper_smoke_result + OUTPUT_VARIABLE helper_smoke_output + ERROR_VARIABLE helper_smoke_error) +if(NOT helper_smoke_result EQUAL 0) + message(FATAL_ERROR + "installed DeepSeek V4.1 containment helper --version failed:\n" + "${helper_smoke_output}${helper_smoke_error}") +endif() +if(NOT helper_smoke_output MATCHES "${DSV41_REVISION}") + message(FATAL_ERROR + "installed DeepSeek V4.1 containment helper reported the wrong revision:\n" + "${helper_smoke_output}") +endif() diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index c49090181710..ea6690d60810 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -1,14 +1,17 @@ #!/usr/bin/env python3 import argparse +import array import ctypes import hashlib import json import math import os import re +import select import selectors import signal +import socket import stat import struct import subprocess @@ -143,8 +146,6 @@ class TraceError(RuntimeError): PROCESS_TREE_TERM_GRACE_SECONDS = 1 WINDOWS_CREATE_SUSPENDED = 0x00000004 WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 -LINUX_PR_SET_CHILD_SUBREAPER = 36 -LINUX_PR_GET_CHILD_SUBREAPER = 37 @dataclass(frozen=True) @@ -171,8 +172,8 @@ def __init__( class _ProcessContainment: process: Any linux_root_pidfd: int | None = None + linux_namespace_pidfd: int | None = None linux_lock_held: bool = False - linux_prior_subreaper: int | None = None linux_exec_released: bool = False test_process_group_id: int | None = None job_handle: int | None = None @@ -196,8 +197,9 @@ class _ContainmentCleanup: quiescence_proven: bool -_LINUX_SUBREAPER_LOCK = threading.Lock() -_LINUX_SUBREAPER_POISONED = False +_LINUX_HELPER_LOCK = threading.Lock() +_LINUX_HELPER_POISONED = False +_LINUX_POISONED_CONTAINMENT: _ProcessContainment | None = None _TEST_PROCESS_GROUP_CONTAINMENT = threading.local() @@ -905,70 +907,105 @@ def _test_only_process_group_containment() -> Iterable[None]: _TEST_PROCESS_GROUP_CONTAINMENT.enabled = previous -class _LinuxForkExecProcess: +class _LinuxNativeHelperProcess: def __init__( self, command: list[str], pid: int, *, - barrier_fd: int, - exec_error_fd: int, + protocol_socket: socket.socket, stdin_fd: int | None, stdout_fd: int | None, stderr_fd: int | None): self.args = command self.pid = pid self.returncode = None - self._barrier_fd = barrier_fd - self._exec_error_fd = exec_error_fd + self.root_pidfd = None + self.namespace_pidfd = None + self.completion_proven = False + self.launch_primary_error = None + self.launch_integrity_failures = [] + self._protocol_socket = protocol_socket self._stdin_fd = stdin_fd self._stdout_fd = stdout_fd self._stderr_fd = stderr_fd + def _receive_protocol(self, expected: bytes, *, receive_pidfd: bool = False) -> None: + item_size = array.array("i").itemsize + data, ancillary, flags, _address = self._protocol_socket.recvmsg( + 128, + socket.CMSG_SPACE(item_size), + getattr(socket, "MSG_CMSG_CLOEXEC", 0), + ) + received = [] + for level, kind, content in ancillary: + if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: + descriptor_bytes = array.array("i") + descriptor_bytes.frombytes(content[:item_size]) + received.extend(descriptor_bytes) + if flags != 0 or data != expected: + for descriptor in received: + os.close(descriptor) + raise TraceError( + f"Linux containment helper protocol expected {expected.decode('ascii')}") + if receive_pidfd: + if len(received) != 1: + for descriptor in received: + os.close(descriptor) + raise TraceError("Linux containment helper did not provide one namespace pidfd") + self.namespace_pidfd = received[0] + elif received: + for descriptor in received: + os.close(descriptor) + raise TraceError("Linux containment helper sent an unexpected descriptor") + def release_exec(self) -> None: - try: - os.write(self._barrier_fd, b"1") - finally: - os.close(self._barrier_fd) - self._barrier_fd = -1 - error_bytes = bytearray() - try: - while True: - chunk = os.read(self._exec_error_fd, 4096) - if not chunk: - break - error_bytes.extend(chunk) - finally: - os.close(self._exec_error_fd) - self._exec_error_fd = -1 - if error_bytes: - self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) - raise OSError(error_bytes.decode("ascii", "strict")) + self._receive_protocol(b"READY") + self._protocol_socket.sendall(b"PREPARE") + self._receive_protocol(b"PREPARED", receive_pidfd=True) + self._protocol_socket.sendall(b"EXEC") + self._receive_protocol(b"RELEASED") def abort_blocked(self) -> _ContainmentCleanup: failures = [] - if self._barrier_fd >= 0: - try: - os.close(self._barrier_fd) - self._barrier_fd = -1 - except BaseException as error: - failures.append(_IntegrityFailure("linux-exec-barrier-close", error)) try: - self.kill() + self._protocol_socket.shutdown(socket.SHUT_RDWR) except BaseException as error: - failures.append(_IntegrityFailure("linux-blocked-child-termination", error)) + failures.append(_IntegrityFailure("linux-helper-protocol-shutdown", error)) + try: + self._protocol_socket.close() + except BaseException as close_error: + failures.append(_IntegrityFailure( + "linux-process-fd-close:protocol", close_error)) + self._protocol_socket = None try: self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) except BaseException as error: failures.append(_IntegrityFailure("linux-blocked-child-reap", error)) + if self._protocol_socket is not None: + try: + self._protocol_socket.close() + except BaseException as close_error: + failures.append(_IntegrityFailure( + "linux-process-fd-close:protocol", close_error)) + self._protocol_socket = None + try: + self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + except BaseException as retry_error: + failures.append(_IntegrityFailure("linux-blocked-child-reap-retry", retry_error)) return _ContainmentCleanup(failures, not failures and self.returncode is not None) def close_streams(self) -> list[_IntegrityFailure]: failures = [] - for attribute in ( - "_barrier_fd", "_exec_error_fd", "_stdin_fd", "_stdout_fd", "_stderr_fd"): + if self._protocol_socket is not None: + try: + self._protocol_socket.close() + except BaseException as error: + failures.append(_IntegrityFailure("linux-process-fd-close:protocol", error)) + self._protocol_socket = None + for attribute in ("_stdin_fd", "_stdout_fd", "_stderr_fd"): descriptor = getattr(self, attribute) - if descriptor is None or descriptor < 0: + if descriptor is None: continue try: os.close(descriptor) @@ -977,7 +1014,7 @@ def close_streams(self) -> list[_IntegrityFailure]: f"linux-process-fd-close:{attribute.removeprefix('_').removesuffix('_fd')}", error, )) - setattr(self, attribute, -1 if attribute in {"_barrier_fd", "_exec_error_fd"} else None) + setattr(self, attribute, None) return failures def poll(self) -> int | None: @@ -1004,7 +1041,7 @@ def wait(self, timeout: float | None = None) -> int: def kill(self) -> None: if self.poll() is None: - os.kill(self.pid, signal.SIGKILL) + raise TraceError("owned Linux helper termination requires its stable pidfd") def communicate( self, @@ -1073,6 +1110,8 @@ def communicate( if not selector.get_map() and self.poll() is None: time.sleep(0.01) self.wait(timeout=0) + self._receive_protocol(b"COMPLETE") + self.completion_proven = True finally: selector.close() self._stdout_fd = None @@ -1100,189 +1139,175 @@ def _linux_child_file_descriptors( return None, descriptor if mode == subprocess.STDOUT and target_fd == 2: return None, subprocess.STDOUT - raise TraceError("Linux fork/exec containment received unsupported stream controls") + raise TraceError("Linux native helper containment received unsupported stream controls") -def _start_linux_blocked_process(command: list[str], launch: dict[str, Any]) -> _LinuxForkExecProcess: +def _all_catchable_signals() -> set[int]: + return { + int(member) for member in signal.valid_signals() + if int(member) not in {signal.SIGKILL, signal.SIGSTOP} + } + + +def _start_linux_native_helper_process( + command: list[str], + launch: dict[str, Any], +) -> _LinuxNativeHelperProcess: controls = dict(launch) - unknown_controls = set(controls) - { - "cwd", "env", "executable", "pass_fds", "stderr", "stdin", "stdout"} - if unknown_controls: - raise TraceError( - f"Linux fork/exec containment received unsupported controls: {sorted(unknown_controls)}") - executable = str(controls.pop("executable", command[0])) + helper_path = str(controls.pop("_containment_helper_path")) + helper_descriptor = int(controls.pop("_containment_helper_descriptor")) + target_executable = str(controls.pop("executable", command[0])) pass_fds = tuple(int(fd) for fd in controls.pop("pass_fds", ())) environment = controls.pop("env", None) working_directory = controls.pop("cwd", None) - opened_descriptors = set() + if working_directory is not None: + raise TraceError("Linux native containment helper does not support cwd") + unknown_controls = set(controls) - {"stderr", "stdin", "stdout"} + if unknown_controls: + raise TraceError( + f"Linux native containment helper received unsupported controls: {sorted(unknown_controls)}") + parent_socket = None + child_socket = None + inheritable_before = {} + stdin_parent = None + stdin_child = None + stdout_parent = None + stdout_child = None + stderr_parent = None + stderr_child = None + process = None + primary_error = None + integrity_failures = [] try: stdin_parent, stdin_child = _linux_child_file_descriptors(controls.pop("stdin", None), 0) - opened_descriptors.update( - descriptor for descriptor in (stdin_parent, stdin_child) - if descriptor is not None and descriptor != subprocess.STDOUT) stdout_parent, stdout_child = _linux_child_file_descriptors(controls.pop("stdout", None), 1) - opened_descriptors.update( - descriptor for descriptor in (stdout_parent, stdout_child) - if descriptor is not None and descriptor != subprocess.STDOUT) stderr_parent, stderr_child = _linux_child_file_descriptors(controls.pop("stderr", None), 2) - opened_descriptors.update( - descriptor for descriptor in (stderr_parent, stderr_child) - if descriptor is not None and descriptor != subprocess.STDOUT) - barrier_read, barrier_write = os.pipe() - opened_descriptors.update((barrier_read, barrier_write)) - error_read, error_write = os.pipe() - opened_descriptors.update((error_read, error_write)) - os.set_inheritable(error_write, False) + parent_socket, child_socket = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET) + parent_socket.settimeout(PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) child_descriptors = [ descriptor for descriptor in (stdin_child, stdout_child, stderr_child) if descriptor is not None and descriptor != subprocess.STDOUT] if any(descriptor <= 2 for descriptor in child_descriptors): - raise TraceError("Linux fork/exec containment requires intact standard descriptors") - except BaseException: - for descriptor in opened_descriptors: - try: - os.close(descriptor) - except OSError: - pass - raise - try: - process_id = os.fork() - except BaseException: - for descriptor in opened_descriptors: - os.close(descriptor) - raise - if process_id == 0: + raise TraceError("Linux native containment helper requires intact standard descriptors") + inherited = {child_socket.fileno(), *pass_fds} + for descriptor in inherited: + inheritable_before[descriptor] = os.get_inheritable(descriptor) + os.set_inheritable(descriptor, True) + helper_argv = [ + helper_path, + "--protocol-fd", str(child_socket.fileno()), + "--expected-parent", str(os.getpid()), + "--exec-path", target_executable, + ] + for descriptor in pass_fds: + helper_argv.extend(("--keep-fd", str(descriptor))) + helper_argv.append("--") + helper_argv.extend(command) + file_actions = [] + for child_fd, target_fd in ( + (stdin_child, 0), (stdout_child, 1), (stderr_child, 2)): + if child_fd == subprocess.STDOUT: + file_actions.append((os.POSIX_SPAWN_DUP2, 1, 2)) + elif child_fd is not None: + file_actions.append((os.POSIX_SPAWN_DUP2, child_fd, target_fd)) + helper_exec_path = f"/proc/self/fd/{helper_descriptor}" + process_id = os.posix_spawn( + helper_exec_path, + helper_argv, + os.environ if environment is None else environment, + file_actions=file_actions, + setsigmask=_all_catchable_signals(), + setsigdef=_all_catchable_signals(), + ) + process = _LinuxNativeHelperProcess( + command, + process_id, + protocol_socket=parent_socket, + stdin_fd=stdin_parent, + stdout_fd=stdout_parent, + stderr_fd=stderr_parent, + ) + parent_socket = None + stdin_parent = None + stdout_parent = None + stderr_parent = None + process.root_pidfd = _linux_open_pidfd(process_id) + except BaseException as error: + primary_error = error + for descriptor, previous in inheritable_before.items(): try: - os.close(barrier_write) - os.close(error_read) - for parent_fd in (stdin_parent, stdout_parent, stderr_parent): - if parent_fd is not None: - os.close(parent_fd) - for child_fd, target_fd in ( - (stdin_child, 0), (stdout_child, 1), (stderr_child, 2)): - if child_fd == subprocess.STDOUT: - os.dup2(1, 2) - elif child_fd is not None: - os.dup2(child_fd, target_fd) - for descriptor in pass_fds: - os.set_inheritable(descriptor, True) - keep = {0, 1, 2, barrier_read, error_write, *pass_fds} - for descriptor_name in os.listdir("/proc/self/fd"): - descriptor = int(descriptor_name) - if descriptor not in keep: - try: - os.close(descriptor) - except OSError: - pass - if os.read(barrier_read, 1) != b"1": - os._exit(126) - os.close(barrier_read) - if working_directory is not None: - os.chdir(working_directory) - os.execve(executable, command, os.environ if environment is None else environment) + os.set_inheritable(descriptor, previous) except BaseException as error: + integrity_failures.append(_IntegrityFailure( + "linux-launch-descriptor-inheritability-restore", error)) + if child_socket is not None: + try: + child_socket.close() + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + "linux-helper-child-protocol-close", error)) + for component, descriptor in ( + ("stdin", stdin_child), + ("stdout", stdout_child), + ("stderr", stderr_child)): + if descriptor is not None and descriptor != subprocess.STDOUT: try: - os.write(error_write, f"{type(error).__name__}: {error}".encode("ascii", "backslashreplace")) - finally: - os._exit(127) - process = _LinuxForkExecProcess( - command, - process_id, - barrier_fd=barrier_write, - exec_error_fd=error_read, - stdin_fd=stdin_parent, - stdout_fd=stdout_parent, - stderr_fd=stderr_parent, - ) - parent_close_failures = [] - for descriptor, component in ( - (barrier_read, "linux-parent-barrier-read-close"), - (error_write, "linux-parent-error-write-close"), - (stdin_child, "linux-parent-stdin-child-close"), - (stdout_child, "linux-parent-stdout-child-close"), - (stderr_child, "linux-parent-stderr-child-close")): - if descriptor is None or descriptor == subprocess.STDOUT: - continue + os.close(descriptor) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"linux-helper-child-{component}-close", error)) + if process is not None: + process.launch_primary_error = primary_error + process.launch_integrity_failures = integrity_failures + return process + if parent_socket is not None: try: - os.close(descriptor) + parent_socket.close() except BaseException as error: - parent_close_failures.append(_IntegrityFailure(component, error)) - if parent_close_failures: - primary_failure = parent_close_failures.pop(0) - cleanup = process.abort_blocked() - parent_close_failures.extend(cleanup.failures) - parent_close_failures.extend(process.close_streams()) + integrity_failures.append(_IntegrityFailure( + "linux-helper-parent-protocol-close", error)) + for component, descriptor in ( + ("stdin", stdin_parent), + ("stdout", stdout_parent), + ("stderr", stderr_parent)): + if descriptor is not None: + try: + os.close(descriptor) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"linux-helper-parent-{component}-close", error)) + if primary_error is None and not integrity_failures: + raise TraceError("Linux native helper launch did not return a process") + if primary_error is None: + primary_error = integrity_failures.pop(0).error + if integrity_failures: raise ExecutionIntegrityError( - f"Linux blocked-child setup primary failure " - f"[{type(primary_failure.error).__name__}: {primary_failure.error}]; " - f"secondary integrity failures: {_format_integrity_failures(parent_close_failures)}", - primary_error=primary_failure.error, - secondary_errors=parent_close_failures, + f"Linux helper launch primary failure " + f"[{type(primary_error).__name__}: {primary_error}]" + + (f"; secondary integrity failures: " + f"{_format_integrity_failures(integrity_failures)}" if integrity_failures else ""), + primary_error=primary_error, + secondary_errors=integrity_failures, quiescence_proven=False, - ) from primary_failure.error - return process - - -def _linux_direct_children() -> set[int]: - children = set() - task_root = Path("/proc/self/task") - if not task_root.is_dir(): - raise TraceError("Linux subreaper containment requires procfs task children") - for task in task_root.iterdir(): - child_file = task / "children" - try: - values = child_file.read_text(encoding="ascii").split() - except FileNotFoundError: - continue - except OSError as error: - raise TraceError(f"cannot read Linux subreaper child ownership: {error}") from error - for value in values: - try: - children.add(int(value)) - except ValueError as error: - raise TraceError("Linux subreaper child ownership is invalid") from error - return children + ) from primary_error + raise primary_error def _linux_task_ids() -> set[int]: task_root = Path("/proc/self/task") if not task_root.is_dir(): - raise TraceError("Linux subreaper containment requires procfs task identities") + raise TraceError("Linux native containment requires procfs task identities") try: return {int(task.name) for task in task_root.iterdir()} except (OSError, ValueError) as error: - raise TraceError(f"cannot read Linux subreaper task identities: {error}") from error - - -def _linux_get_child_subreaper() -> int: - libc = ctypes.CDLL(None, use_errno=True) - enabled = ctypes.c_int() - if libc.prctl(LINUX_PR_GET_CHILD_SUBREAPER, ctypes.byref(enabled), 0, 0, 0) != 0: - error_number = ctypes.get_errno() - raise TraceError(f"cannot read Linux child subreaper: {os.strerror(error_number)}") - if enabled.value not in (0, 1): - raise TraceError(f"invalid Linux child subreaper state {enabled.value}") - return enabled.value - - -def _linux_set_child_subreaper(value: int) -> None: - if value not in (0, 1): - raise TraceError(f"invalid Linux child subreaper state {value}") - libc = ctypes.CDLL(None, use_errno=True) - if libc.prctl(LINUX_PR_SET_CHILD_SUBREAPER, value, 0, 0, 0) != 0: - error_number = ctypes.get_errno() - raise TraceError(f"cannot set Linux child subreaper: {os.strerror(error_number)}") - actual = _linux_get_child_subreaper() - if actual != value: - raise TraceError( - f"Linux child subreaper verification failed: expected {value}, got {actual}") + raise TraceError(f"cannot read Linux native containment task identities: {error}") from error def _linux_open_pidfd(process_id: int) -> int: opener = getattr(os, "pidfd_open", None) sender = getattr(signal, "pidfd_send_signal", None) if opener is None or sender is None: - raise TraceError("Linux subreaper containment requires pidfd signaling") + raise TraceError("Linux native containment requires pidfd signaling") try: return int(opener(process_id, 0)) except ProcessLookupError: @@ -1293,28 +1318,20 @@ def _linux_open_pidfd(process_id: int) -> int: def _linux_require_pidfd_support() -> None: if getattr(os, "pidfd_open", None) is None or getattr(signal, "pidfd_send_signal", None) is None: - raise TraceError("Linux subreaper containment requires pidfd signaling") + raise TraceError("Linux native containment requires pidfd signaling") def _linux_signal_pidfd(pidfd: int, requested_signal: int) -> None: sender = getattr(signal, "pidfd_send_signal", None) if sender is None: - raise TraceError("Linux subreaper containment requires pidfd signaling") + raise TraceError("Linux native containment requires pidfd signaling") sender(pidfd, requested_signal) -def _linux_reap_owned_descendants(root_pid: int) -> list[_IntegrityFailure]: - failures = [] - for process_id in sorted(_linux_direct_children()): - if process_id == root_pid: - continue - try: - os.waitpid(process_id, os.WNOHANG) - except ChildProcessError: - continue - except BaseException as error: - failures.append(_IntegrityFailure("linux-descendant-reap", error)) - return failures +def _linux_pidfd_has_exited(pidfd: int) -> bool: + poller = select.poll() + poller.register(pidfd, select.POLLIN) + return bool(poller.poll(0)) def _linux_signal_owned_children( @@ -1322,101 +1339,89 @@ def _linux_signal_owned_children( requested_signal: int, ) -> list[_IntegrityFailure]: failures = [] - root_pid = containment.process.pid - try: - process_ids = _linux_direct_children() - except BaseException as error: - return [_IntegrityFailure("linux-child-ownership", error)] - for process_id in sorted(process_ids): - pidfd = containment.linux_root_pidfd if process_id == root_pid else None - close_pidfd = False + for component, pidfd in ( + ("linux-namespace-termination", containment.linux_namespace_pidfd), + ("linux-helper-termination", containment.linux_root_pidfd)): + if pidfd is None: + continue try: - if pidfd is None: - pidfd = _linux_open_pidfd(process_id) - close_pidfd = True _linux_signal_pidfd(pidfd, requested_signal) except ProcessLookupError: pass except BaseException as error: - failures.append(_IntegrityFailure("linux-process-termination", error)) - finally: - if close_pidfd and pidfd is not None: - try: - os.close(pidfd) - except BaseException as error: - failures.append(_IntegrityFailure("linux-pidfd-close", error)) + failures.append(_IntegrityFailure(component, error)) return failures -def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: - global _LINUX_SUBREAPER_POISONED - if not _LINUX_SUBREAPER_LOCK.acquire(blocking=False): - raise TraceError("Linux subreaper containment is already active") +def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + global _LINUX_HELPER_POISONED + if not _LINUX_HELPER_LOCK.acquire(blocking=False): + raise TraceError("Linux native containment helper is already active") + if _LINUX_HELPER_POISONED: + _LINUX_HELPER_LOCK.release() + raise TraceError("Linux native containment helper supervisor is not reusable") process = None pidfd = None - prior_subreaper = None exec_released = False try: - if _LINUX_SUBREAPER_POISONED: - raise TraceError("Linux subreaper containment supervisor is not reusable") - prior_subreaper = _linux_get_child_subreaper() - if prior_subreaper != 1: - _linux_set_child_subreaper(1) _linux_require_pidfd_support() if len(_linux_task_ids()) != 1: - raise TraceError("Linux subreaper containment requires a single-threaded supervisor") - if _linux_direct_children(): - raise TraceError("Linux subreaper containment process owns unrelated children") - process = _start_linux_blocked_process(command, launch) - pidfd = _linux_open_pidfd(process.pid) + raise TraceError("Linux native containment requires a single-threaded Python supervisor") + process = _start_linux_native_helper_process(command, launch) + pidfd = process.root_pidfd containment = _ProcessContainment( process=process, linux_root_pidfd=pidfd, linux_lock_held=True, - linux_prior_subreaper=prior_subreaper, ) + if process.launch_primary_error is not None or process.launch_integrity_failures: + launch_primary = process.launch_primary_error + launch_failures = list(process.launch_integrity_failures) + if launch_primary is None: + launch_primary = launch_failures.pop(0).error + raise ExecutionIntegrityError( + f"Linux helper launch primary failure " + f"[{type(launch_primary).__name__}: {launch_primary}]" + + (f"; secondary integrity failures: " + f"{_format_integrity_failures(launch_failures)}" if launch_failures else ""), + primary_error=launch_primary, + secondary_errors=launch_failures, + quiescence_proven=False, + ) from launch_primary + if pidfd is None: + raise TraceError("Linux native containment helper identity is unavailable") + process.release_exec() + if process.namespace_pidfd is None or _linux_pidfd_has_exited(process.namespace_pidfd): + raise TraceError("Linux target PID namespace authority is unavailable before exec") + containment.linux_namespace_pidfd = process.namespace_pidfd exec_released = True containment.linux_exec_released = True - process.release_exec() return containment except BaseException as primary_error: failures = [] - startup_quiescence_proven = False if isinstance(primary_error, ExecutionIntegrityError): failures.extend(primary_error.secondary_errors) - startup_quiescence_proven = primary_error.quiescence_proven primary_error = primary_error.primary_error or primary_error if process is not None: failed_containment = _ProcessContainment( process=process, linux_root_pidfd=pidfd, + linux_namespace_pidfd=process.namespace_pidfd, linux_lock_held=True, - linux_prior_subreaper=prior_subreaper, linux_exec_released=exec_released, ) - if exec_released: + if process.namespace_pidfd is not None or exec_released: cleanup = _terminate_process_tree(failed_containment) - failures.extend(cleanup.failures) - startup_quiescence_proven = cleanup.quiescence_proven else: cleanup = process.abort_blocked() - failures.extend(cleanup.failures) - startup_quiescence_proven = cleanup.quiescence_proven + failures.extend(cleanup.failures) close_failures = _close_process_containment( failed_containment, - quiescence_proven=startup_quiescence_proven, + quiescence_proven=cleanup.quiescence_proven, ) failures.extend(close_failures) - startup_quiescence_proven = ( - pidfd is not None and startup_quiescence_proven and not close_failures) else: - if prior_subreaper is not None: - try: - _linux_set_child_subreaper(prior_subreaper) - except BaseException as error: - _LINUX_SUBREAPER_POISONED = True - failures.append(_IntegrityFailure("linux-subreaper-restore", error)) - _LINUX_SUBREAPER_LOCK.release() + _LINUX_HELPER_LOCK.release() if failures: raise ExecutionIntegrityError( f"Linux containment startup primary failure " @@ -1424,7 +1429,7 @@ def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) - f"secondary integrity failures: {_format_integrity_failures(failures)}", primary_error=primary_error, secondary_errors=failures, - quiescence_proven=startup_quiescence_proven, + quiescence_proven=False, ) from primary_error if process is not None: raise ExecutionIntegrityError( @@ -1432,7 +1437,7 @@ def _start_linux_subreaper_process(command: list[str], launch: dict[str, Any]) - f"[{type(primary_error).__name__}: {primary_error}]", primary_error=primary_error, secondary_errors=[], - quiescence_proven=startup_quiescence_proven, + quiescence_proven=False, ) from primary_error raise @@ -1441,7 +1446,7 @@ def _start_contained_process(command: list[str], launch: dict[str, Any]) -> _Pro if sys.platform == "win32": return _start_windows_job_process(command, launch) if sys.platform == "linux": - return _start_linux_subreaper_process(command, launch) + return _start_linux_native_helper(command, launch) if sys.platform == "darwin" and getattr(_TEST_PROCESS_GROUP_CONTAINMENT, "enabled", False): launch["start_new_session"] = True process = subprocess.Popen(command, **launch) @@ -1497,10 +1502,10 @@ def _process_tree_is_quiescent(containment: _ProcessContainment) -> bool: if containment.job_handle is not None: return _windows_job_active_processes(containment.job_handle) == 0 if containment.linux_lock_held: - reap_failures = _linux_reap_owned_descendants(containment.process.pid) - if reap_failures: - raise reap_failures[0].error - return containment.process.poll() is not None and not _linux_direct_children() + helper_exited = containment.process.poll() is not None + if containment.linux_namespace_pidfd is None: + return helper_exited and not containment.linux_exec_released + return helper_exited and _linux_pidfd_has_exited(containment.linux_namespace_pidfd) if containment.test_process_group_id is not None: return not _test_process_group_exists(containment.test_process_group_id) raise TraceError("process containment identity is missing") @@ -1539,23 +1544,7 @@ def _terminate_process_tree(containment: _ProcessContainment) -> _ContainmentCle failures.append(_IntegrityFailure("direct-child-reap", error)) except BaseException as error: failures.append(_IntegrityFailure("direct-child-reap", error)) - descendant_term_deadline = min( - deadline, time.monotonic() + PROCESS_TREE_TERM_GRACE_SECONDS) - failures.extend(_linux_signal_owned_children(containment, signal.SIGTERM)) - try: - _wait_for_process_tree_quiescence(containment, descendant_term_deadline) - except BaseException: - failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) - while time.monotonic() < deadline: - failures.extend(_linux_reap_owned_descendants(process.pid)) - try: - if _process_tree_is_quiescent(containment): - break - except BaseException as error: - failures.append(_IntegrityFailure("linux-child-ownership", error)) - break - failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) - time.sleep(0.01) + failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) elif containment.test_process_group_id is not None: try: if _test_process_group_exists(containment.test_process_group_id): @@ -1585,6 +1574,12 @@ def _terminate_process_tree(containment: _ProcessContainment) -> _ContainmentCle _wait_for_process_tree_quiescence(containment, deadline) except BaseException as error: failures.append(_IntegrityFailure("process-tree-quiescence", error)) + if containment.linux_exec_released and isinstance( + process, _LinuxNativeHelperProcess) and not process.completion_proven: + failures.append(_IntegrityFailure( + "linux-helper-completion", + TraceError("Linux native helper did not prove containment teardown"), + )) quiescence_proven = not failures if quiescence_proven: try: @@ -1644,10 +1639,18 @@ def _close_process_containment( *, quiescence_proven: bool, ) -> list[_IntegrityFailure]: - global _LINUX_SUBREAPER_POISONED + global _LINUX_HELPER_POISONED + global _LINUX_POISONED_CONTAINMENT if containment is None: return [] failures = [] + if containment.linux_lock_held and not quiescence_proven: + failures.append(_IntegrityFailure( + "linux-helper-teardown", + TraceError("Linux native helper cannot be released before full tree quiescence"))) + _LINUX_HELPER_POISONED = True + _LINUX_POISONED_CONTAINMENT = containment + return failures if containment.job_handle is not None: try: if not _windows_kernel32().CloseHandle(containment.job_handle): @@ -1659,33 +1662,29 @@ def _close_process_containment( _close_windows_process_handle(containment.process) except BaseException as error: failures.append(_IntegrityFailure("windows-process-handle-close", error)) + linux_failure_count = len(failures) if containment.linux_root_pidfd is not None: try: os.close(containment.linux_root_pidfd) containment.linux_root_pidfd = None + if isinstance(containment.process, _LinuxNativeHelperProcess): + containment.process.root_pidfd = None except BaseException as error: failures.append(_IntegrityFailure("linux-root-pidfd-close", error)) - if isinstance(containment.process, _LinuxForkExecProcess): + if containment.linux_namespace_pidfd is not None: + try: + os.close(containment.linux_namespace_pidfd) + containment.linux_namespace_pidfd = None + except BaseException as error: + failures.append(_IntegrityFailure("linux-namespace-pidfd-close", error)) + if isinstance(containment.process, _LinuxNativeHelperProcess): failures.extend(containment.process.close_streams()) if containment.linux_lock_held: - if not quiescence_proven: - failures.append(_IntegrityFailure( - "linux-subreaper-restore", - TraceError("Linux child-subreaper state cannot be restored before full tree quiescence"))) - _LINUX_SUBREAPER_POISONED = True - elif containment.linux_prior_subreaper is None: - failures.append(_IntegrityFailure( - "linux-subreaper-restore", - TraceError("Linux child-subreaper prior state is missing"))) - _LINUX_SUBREAPER_POISONED = True - else: - try: - _linux_set_child_subreaper(containment.linux_prior_subreaper) - except BaseException as error: - _LINUX_SUBREAPER_POISONED = True - failures.append(_IntegrityFailure("linux-subreaper-restore", error)) + if len(failures) != linux_failure_count: + _LINUX_HELPER_POISONED = True + _LINUX_POISONED_CONTAINMENT = containment containment.linux_lock_held = False - _LINUX_SUBREAPER_LOCK.release() + _LINUX_HELPER_LOCK.release() return failures @@ -1787,6 +1786,23 @@ def run_approved_executable( label=label, executable=True, ) + containment_helper: tuple[ExecutableFileReceipt, int] | None = None + if sys.platform == "linux": + helper_policy = _validate_containment_helper_policy( + runtime_policy.get("containment_helper"), + install_root=runtime_policy["install_root"], + revision=runtime_policy["revision"], + label=label, + ) + containment_helper = _approved_file_identity( + Path(helper_policy["path"]), + install_root=Path(runtime_policy["install_root"]), + expected_owner_uid=runtime_policy["install_owner_uid"], + expected_path=helper_policy["path"], + expected_sha256=helper_policy["sha256"], + label=f"{label} containment helper", + executable=True, + ) runtime_files: list[tuple[ExecutableFileReceipt, int]] = [] containment = None result = None @@ -1808,18 +1824,29 @@ def run_approved_executable( executable=False, )) verify_approved_executable_identity(path, identity, label=label) + if containment_helper is not None: + verify_approved_executable_identity( + Path(containment_helper[0].path), + containment_helper[0], + label=f"{label} containment helper", + ) for runtime_identity, _runtime_descriptor in runtime_files: verify_approved_executable_identity( Path(runtime_identity.path), runtime_identity, label=f"{label} runtime component", ) - retained_descriptors = (descriptor, *(item[1] for item in runtime_files)) + retained_descriptors = ( + descriptor, + *(item[1] for item in runtime_files), + ) launch = dict(kwargs) if sys.platform != "win32": launch["pass_fds"] = retained_descriptors if sys.platform == "linux": launch["executable"] = f"/proc/self/fd/{descriptor}" + launch["_containment_helper_path"] = containment_helper[0].path + launch["_containment_helper_descriptor"] = containment_helper[1] contained = _run_contained_process( command, label=label, @@ -1862,6 +1889,34 @@ def run_approved_executable( verify_approved_executable_identity(path, identity, label=label) except BaseException as error: integrity_failures.append(_IntegrityFailure("executable-path-root", error)) + if containment_helper is not None: + helper_identity, helper_descriptor = containment_helper + try: + helper_after = os.fstat(helper_descriptor) + if ( + helper_after.st_dev, + helper_after.st_ino, + helper_after.st_size, + helper_after.st_mtime_ns, + helper_after.st_ctime_ns, + ) != ( + helper_identity.device, + helper_identity.inode, + helper_identity.byte_count, + helper_identity.modified_ns, + helper_identity.changed_ns, + ): + raise TraceError(f"{label} containment helper descriptor changed during execution") + except BaseException as error: + integrity_failures.append(_IntegrityFailure("containment-helper-descriptor", error)) + try: + verify_approved_executable_identity( + Path(helper_identity.path), + helper_identity, + label=f"{label} containment helper", + ) + except BaseException as error: + integrity_failures.append(_IntegrityFailure("containment-helper-path-root", error)) for runtime_identity, runtime_descriptor in runtime_files: try: runtime_after = os.fstat(runtime_descriptor) @@ -1903,21 +1958,28 @@ def run_approved_executable( else: integrity_failures.append(_IntegrityFailure("launcher-orchestration", error)) finally: + teardown_failures = [] for _runtime_identity, runtime_descriptor in runtime_files: try: os.close(runtime_descriptor) except BaseException as error: - integrity_failures.append(_IntegrityFailure("runtime-descriptor-close", error)) + teardown_failures.append(_IntegrityFailure("runtime-descriptor-close", error)) + if containment_helper is not None: + try: + os.close(containment_helper[1]) + except BaseException as error: + teardown_failures.append(_IntegrityFailure("containment-helper-descriptor-close", error)) try: os.close(descriptor) except BaseException as error: - integrity_failures.append(_IntegrityFailure("executable-descriptor-close", error)) + teardown_failures.append(_IntegrityFailure("executable-descriptor-close", error)) containment_failures = _close_process_containment( containment, quiescence_proven=quiescence_proven, ) - integrity_failures.extend(containment_failures) - if containment_failures: + teardown_failures.extend(containment_failures) + integrity_failures.extend(teardown_failures) + if teardown_failures: quiescence_proven = False if result is not None and primary_error is None: try: @@ -1944,8 +2006,9 @@ def run_approved_executable( def install_trust_evidence( executable: ExecutableFileReceipt, runtime_files: list[ExecutableFileReceipt], + additional_files: tuple[ExecutableFileReceipt, ...] = (), ) -> dict[str, Any]: - files = [executable, *runtime_files] + files = [executable, *runtime_files, *additional_files] if any(item.install_root != executable.install_root or item.owner_uid != executable.owner_uid for item in files): raise TraceError("approved install trust evidence spans multiple roots or owners") directories: dict[str, tuple[str, int, int, int, int]] = {} @@ -2084,6 +2147,14 @@ def validate_install_trust_evidence( for component in policy["runtime_receipt"]["components"] }, } + if "containment_helper" in policy: + helper = _validate_containment_helper_policy( + policy["containment_helper"], + install_root=install_root, + revision=policy["revision"], + label="external approval", + ) + expected_files[helper["path"]] = helper["sha256"] observed_files = {item["path"]: item["sha256"] for item in files} if observed_files != expected_files: raise TraceError("install trust files differ from external approval") @@ -2295,6 +2366,51 @@ def _approval_digest(kind: str, approval_id: str, policy: dict[str, Any]) -> str return sha256_bytes(canonical_json(record).encode("ascii")) +def _validate_containment_helper_policy( + record: object, + *, + install_root: str, + revision: str, + label: str, +) -> dict[str, Any]: + if not isinstance(record, dict): + raise TraceError(f"{label} containment helper receipt is missing") + _require_exact_keys( + record, + {"format", "version", "revision", "filename", "sha256"}, + f"{label} containment helper receipt", + ) + if record["format"] != "dsv41-containment-helper" or record["version"] != 1 or ( + record["revision"] != revision) or ( + record["filename"] != "llama-deepseek-v41-containment-helper") or re.fullmatch( + r"[0-9a-f]{64}", record.get("sha256", "")) is None: + raise TraceError(f"{label} containment helper receipt is invalid") + result = dict(record) + result["path"] = f"{install_root}/bin/{record['filename']}" + return result + + +def approved_containment_helper_identity( + policy: dict[str, Any], + *, + label: str, +) -> ExecutableFileReceipt: + helper = _validate_containment_helper_policy( + policy.get("containment_helper"), + install_root=policy["install_root"], + revision=policy["revision"], + label=label, + ) + return approved_executable_identity( + Path(helper["path"]), + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=helper["path"], + expected_sha256=helper["sha256"], + label=f"{label} containment helper", + ) + + def candidate_exporter_approval( approval_id: str, *, @@ -2315,6 +2431,7 @@ def candidate_exporter_approval( "install_owner_uid", "executable_path", "executable_sha256", + "containment_helper", "runtime_profile", "runtime_receipt", }, @@ -2335,6 +2452,12 @@ def candidate_exporter_approval( policy["executable_path"], "candidate exporter approval executable path") if executable_path != f"{install_root}/bin/llama-deepseek-v41-trace": raise TraceError("candidate exporter approval executable path is outside its install policy") + _validate_containment_helper_policy( + policy["containment_helper"], + install_root=install_root, + revision=policy["revision"], + label="candidate exporter approval", + ) profile = policy["runtime_profile"] if not isinstance(profile, dict): raise TraceError("candidate exporter approval runtime profile is invalid") @@ -2537,6 +2660,7 @@ def prompt_builder_approval( "install_owner_uid", "executable_path", "executable_sha256", + "containment_helper", "source_root", "runtime_receipt", "model_sha256", @@ -2561,6 +2685,12 @@ def prompt_builder_approval( source_root = _approval_path(policy["source_root"], "prompt builder approval source root") if executable_path != f"{install_root}/bin/llama-deepseek-v41-prompt-builder": raise TraceError("prompt builder approval executable path is outside its install policy") + _validate_containment_helper_policy( + policy["containment_helper"], + install_root=install_root, + revision=policy["revision"], + label="prompt builder approval", + ) profile = policy["runtime_profile"] if not isinstance(profile, dict): raise TraceError("prompt builder approval runtime profile is invalid") From 90ea4d3c99e2e0ca7f949df9f25446f92a1cfc6d Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 17:27:01 -0700 Subject: [PATCH 43/56] trace : isolate Linux target privileges Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 273 +++++++- tools/deepseek-v41-trace/README.md | 2 +- .../linux-containment-helper.cpp | 657 +++++++++++++++++- tools/deepseek-v41-trace/trace_format.py | 1 + 4 files changed, 889 insertions(+), 44 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 0654b486562a..6b2029260a75 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2433,6 +2433,7 @@ def test_linux_native_helper_boundary_precedes_target_release(self) -> None: Path(__file__).parents[1] / "tools/deepseek-v41-trace/linux-containment-helper.cpp" ).read_text(encoding="ascii") + helper_main = helper_source[helper_source.index("int run_linux_helper"):] self.assertNotIn("os.fork", start_source + spawn_source) self.assertIn("os.posix_spawn", spawn_source) self.assertLess(spawn_source.index("os.posix_spawn"), spawn_source.index("_linux_open_pidfd")) @@ -2441,17 +2442,17 @@ def test_linux_native_helper_boundary_precedes_target_release(self) -> None: for flag in ("CLONE_NEWUSER", "CLONE_NEWPID", "CLONE_NEWNS", "CLONE_PIDFD"): self.assertIn(flag, helper_source) self.assertLess( - helper_source.index("namespace_owner owned_namespace"), - helper_source.index('"uid_map"'), + helper_main.index("namespace_owner owned_namespace"), + helper_main.index('"uid_map"'), ) self.assertIn('"uid_map"', helper_source) self.assertIn('mount("proc", "/proc", "proc"', helper_source) self.assertLess( helper_source.index("target PID namespace did not bind helper lifetime"), - helper_source.index("send_pidfd(config.protocol_fd"), + helper_source.index('send_descriptor(config.protocol_fd, \"PREPARED\"'), ) self.assertLess( - helper_source.index("send_pidfd(config.protocol_fd"), + helper_source.index('send_descriptor(config.protocol_fd, \"PREPARED\"'), helper_source.index('!= \"EXEC\"'), ) @@ -2481,12 +2482,106 @@ def test_linux_native_helper_inherited_signal_defenses_are_explicit(self) -> Non ).read_text(encoding="ascii") self.assertIn("setsigmask=_all_catchable_signals()", spawn_source) self.assertIn("setsigdef=_all_catchable_signals()", spawn_source) + self.assertIn("setsid=True", spawn_source) self.assertIn("require_initial_signal_state();", helper_source) self.assertLess( helper_source.index("require_initial_signal_state();"), helper_source.index('send_packet(config.protocol_fd, \"READY\")'), ) + def test_linux_native_helper_source_isolates_target_privileges_before_exec(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + bootstrap_start = helper_source.index("[[noreturn]] void run_target_bootstrap") + init_start = helper_source.index("[[noreturn]] void run_namespace_init") + bootstrap_source = helper_source[bootstrap_start:init_start] + init_source = helper_source[init_start:helper_source.index("int run_linux_helper")] + self.assertLess( + bootstrap_source.index("set_parent_death(1);"), + bootstrap_source.index("make_isolated_session();"), + ) + self.assertLess( + bootstrap_source.index("make_isolated_session();"), + bootstrap_source.index("drop_target_privileges();"), + ) + self.assertLess( + bootstrap_source.index("drop_target_privileges();"), + bootstrap_source.index("execve("), + ) + self.assertLess( + bootstrap_source.index('send_descriptor(security_fd, \"FILTER\", listener);'), + bootstrap_source.index("verify_target_attack_denials();"), + ) + self.assertLess( + bootstrap_source.index("verify_target_attack_denials();"), + bootstrap_source.index('send_packet(security_fd, \"VERIFIED\");'), + ) + self.assertLess( + bootstrap_source.index('receive_packet(security_fd) != \"GO\"'), + bootstrap_source.index("execve("), + ) + for required in ( + "PR_SET_DUMPABLE", + "PR_SET_PTRACER", + "PR_SET_SECUREBITS", + "SECBIT_NOROOT_LOCKED", + "SECBIT_NO_SETUID_FIXUP_LOCKED", + "SECBIT_KEEP_CAPS_LOCKED", + "SECBIT_NO_CAP_AMBIENT_RAISE_LOCKED", + "PR_CAPBSET_DROP", + "PR_CAP_AMBIENT_CLEAR_ALL", + "SYS_capset", + "PR_SET_NO_NEW_PRIVS", + "SECCOMP_RET_USER_NOTIF", + "SECCOMP_FILTER_FLAG_NEW_LISTENER", + "SECCOMP_IOCTL_NOTIF_RECV", + "target attempted a forbidden lifecycle operation", + "__NR_ptrace", + "__NR_process_vm_readv", + "__NR_process_vm_writev", + "__NR_kill", + "__NR_setresuid", + "__NR_setpgid", + "__NR_setsid"): + self.assertIn(required, helper_source) + self.assertIn("target_arguments.flags = CLONE_NEWUSER | CLONE_PIDFD;", helper_source) + self.assertIn('"65534 0 1\\n"', helper_source) + self.assertLess( + helper_source.index("protect_namespace_init();"), + helper_source.index('write_all(ready_fd, \"R\", 1);'), + ) + self.assertLess( + init_source.index('target_ready != \'I\''), + init_source.index('write_all(ready_fd, \"I\", 1);'), + ) + self.assertLess( + init_source.index('receive_descriptor(target_security[0], \"FILTER\")'), + init_source.index("verify_target_isolation_probes(target_listener);"), + ) + self.assertLess( + init_source.index("verify_target_isolation_probes(target_listener);"), + init_source.index('send_packet(target_security[0], \"GO\");'), + ) + self.assertLess( + init_source.index('write_all(ready_fd, \"I\", 1);'), + init_source.index("wait_for_isolated_target(owned_target, target_listener)"), + ) + self.assertLess( + init_source.index('write_all(ready_fd, \"C\", 1);'), + init_source.index("_exit(wait_status_exit_code(target_status));"), + ) + helper_main = helper_source[helper_source.index("int run_linux_helper"):] + self.assertLess( + helper_main.index("owned_namespace.wait();"), + helper_main.index("target namespace teardown did not complete"), + ) + self.assertLess( + helper_main.index("target namespace teardown did not complete"), + helper_main.index('send_packet(config.protocol_fd, \"COMPLETE\");'), + ) + def test_linux_native_helper_without_pidfd_support_fails_before_spawn(self) -> None: lock = mock.Mock() lock.acquire.return_value = True @@ -2725,6 +2820,9 @@ def test_linux_native_helper_source_binds_supervisor_death_and_namespace_lifecyc self.assertIn("kill(-1, SIGKILL)", helper_source) self.assertIn("getppid() != expected_parent", helper_source) self.assertIn("getppid() != 0", helper_source) + self.assertGreaterEqual(helper_source.count("make_isolated_session();"), 2) + self.assertIn("require_parent_death(0);", helper_source) + self.assertIn("require_parent_death(1);", helper_source) @unittest.skipUnless( sys.platform == "linux" and os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER"), @@ -2743,17 +2841,26 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: listener.listen(2) listener.settimeout(10) target_source = ( - "import array,os,socket,sys,time\n" - "def report(label):\n" + "import array,ctypes,json,os,socket,sys,time\n" + "libc=ctypes.CDLL(None,use_errno=True)\n" + "def report(label,payload=None):\n" " s=socket.socket(socket.AF_UNIX,socket.SOCK_SEQPACKET)\n" " s.connect(sys.argv[1])\n" " fd=os.pidfd_open(os.getpid())\n" " rights=array.array('i',[fd])\n" - " s.sendmsg([label.encode('ascii')],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,rights)])\n" + " data=label if payload is None else label+':'+json.dumps(payload,sort_keys=True)\n" + " s.sendmsg([data.encode('ascii')],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,rights)])\n" " os.close(fd);s.close()\n" - "report('target')\n" + "status={line.split(':',1)[0]:line.split(':',1)[1].strip()" + " for line in open('/proc/self/status',encoding='ascii') if ':' in line}\n" + "state={'uids':list(os.getresuid()),'gids':list(os.getresgid()),'groups':os.getgroups()," + "'sid':os.getsid(0),'pgrp':os.getpgrp(),'pid':os.getpid()," + "'caps':[status[name] for name in ('CapInh','CapPrm','CapEff','CapBnd','CapAmb')]," + "'no_new_privs':status['NoNewPrivs'],'seccomp':status['Seccomp']," + "'securebits':libc.prctl(27,0,0,0,0)}\n" + "report('target',state)\n" "if os.fork()==0:\n" - " os.setsid();report('descendant')\n" + " time.sleep(.2);report('descendant')\n" " while True: time.sleep(1)\n" "while True: time.sleep(1)\n" ) @@ -2770,7 +2877,7 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: "'_containment_helper_descriptor':helper_fd," "'stdout':-3,'stderr':-3}\n" "containment=trace._start_linux_native_helper(" - "[sys.executable,'-c',sys.argv[2],sys.argv[3]],launch)\n" + "[sys.executable,'-c',sys.argv[2],sys.argv[3],str(os.getpgrp())],launch)\n" "while True: time.sleep(1)\n" ) environment = { @@ -2787,6 +2894,7 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: pidfds = [] try: labels = set() + target_state = None while len(pidfds) < 2: try: connection, _address = listener.accept() @@ -2804,9 +2912,22 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: descriptors.frombytes(content[:descriptors.itemsize]) self.assertEqual(len(descriptors), 1) - labels.add(data.decode("ascii")) + decoded = data.decode("ascii") + label, separator, payload = decoded.partition(":") + labels.add(label) + if separator: + target_state = json.loads(payload) pidfds.append(descriptors[0]) self.assertEqual(labels, {"target", "descendant"}) + self.assertEqual(target_state["uids"], [65534] * 3) + self.assertEqual(target_state["gids"], [65534] * 3) + self.assertEqual(target_state["groups"], []) + self.assertEqual(target_state["sid"], target_state["pid"]) + self.assertEqual(target_state["pgrp"], target_state["pid"]) + self.assertEqual(target_state["caps"], ["0000000000000000"] * 5) + self.assertEqual(target_state["no_new_privs"], "1") + self.assertEqual(target_state["seccomp"], "2") + self.assertEqual(target_state["securebits"], 239) supervisor.kill() supervisor.wait(timeout=5) deadline = time.monotonic() + 5 @@ -2825,6 +2946,136 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: supervisor.wait(timeout=5) listener.close() + @unittest.skipUnless( + sys.platform == "linux" and os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER"), + "native Linux containment helper was not executed on this host", + ) + def test_linux_native_helper_forbidden_operations_kill_namespace(self) -> None: + import array + import socket + + helper = Path(os.environ["DSV41_NATIVE_CONTAINMENT_HELPER"]).resolve(strict=True) + target_source = ( + "import array,ctypes,os,socket,sys,time\n" + "libc=ctypes.CDLL(None,use_errno=True)\n" + "s=socket.socket(socket.AF_UNIX,socket.SOCK_SEQPACKET)\n" + "s.connect(sys.argv[1])\n" + "fd=os.pidfd_open(os.getpid())\n" + "rights=array.array('i',[fd])\n" + "s.sendmsg([b'ready'],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,rights)])\n" + "os.close(fd);s.close();time.sleep(.1)\n" + "attack=sys.argv[2]\n" + "if attack=='ptrace': libc.ptrace(16,1,0,0)\n" + "elif attack=='ptrace_poke': libc.ptrace(5,1,0,0)\n" + "elif attack=='process_vm_readv': libc.process_vm_readv(1,0,0,0,0,0)\n" + "elif attack=='process_vm_writev': libc.process_vm_writev(1,0,0,0,0,0)\n" + "elif attack=='pdeathsig': libc.prctl(1,0,0,0,0)\n" + "elif attack=='pid1_stop': libc.kill(1,19)\n" + "elif attack=='outer_group': libc.kill(-int(sys.argv[3]),0)\n" + "elif attack=='setuid': libc.setresuid(0,0,0)\n" + "elif attack=='setpgid': libc.setpgid(0,0)\n" + "elif attack=='setsid': libc.setsid()\n" + "else: raise SystemExit(91)\n" + "open(sys.argv[4],'w',encoding='ascii').write('survived')\n" + "while True: time.sleep(1)\n" + ) + supervisor_source = ( + "import os,sys\n" + "from pathlib import Path\n" + "import trace_format as trace\n" + "helper=Path(sys.argv[1])\n" + "target=os.open(sys.executable,os.O_RDONLY)\n" + "helper_fd=os.open(helper,os.O_RDONLY)\n" + "launch={'executable':f'/proc/self/fd/{target}'," + "'pass_fds':(target,)," + "'_containment_helper_path':f'/proc/self/fd/{helper_fd}'," + "'_containment_helper_descriptor':helper_fd," + "'stdout':-3,'stderr':-3}\n" + "containment=trace._start_linux_native_helper(" + "[sys.executable,'-c',sys.argv[2],sys.argv[3],sys.argv[4]," + "str(os.getpgrp()),sys.argv[5]],launch)\n" + "containment.process.communicate(timeout=10)\n" + "raise SystemExit(containment.process.returncode)\n" + ) + environment = { + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(Path(__file__).parents[1] / "tools/deepseek-v41-trace"), + } + attacks = ( + "ptrace", + "ptrace_poke", + "process_vm_readv", + "process_vm_writev", + "pdeathsig", + "pid1_stop", + "outer_group", + "setuid", + "setpgid", + "setsid", + ) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for attack in attacks: + with self.subTest(attack=attack): + socket_path = root / f"{attack}.sock" + marker = root / f"{attack}.survived" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + listener.bind(str(socket_path)) + listener.listen(1) + listener.settimeout(10) + supervisor = subprocess.Popen( + [ + sys.executable, + "-c", + supervisor_source, + str(helper), + target_source, + str(socket_path), + attack, + str(marker), + ], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + target_pidfd = None + try: + try: + connection, _address = listener.accept() + except TimeoutError: + if supervisor.poll() is not None: + stderr = supervisor.stderr.read() + self.skipTest( + f"native PID namespace unavailable: {stderr.strip()}") + raise + with connection: + descriptors = array.array("i") + data, ancillary, flags, _address = connection.recvmsg( + 32, socket.CMSG_SPACE(descriptors.itemsize)) + self.assertEqual((data, flags), (b"ready", 0)) + for level, kind, content in ancillary: + if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: + descriptors.frombytes(content[:descriptors.itemsize]) + self.assertEqual(len(descriptors), 1) + target_pidfd = descriptors[0] + self.assertEqual(supervisor.wait(timeout=10), 125) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and not trace._linux_pidfd_has_exited( + target_pidfd): + time.sleep(0.02) + self.assertTrue(trace._linux_pidfd_has_exited(target_pidfd)) + self.assertFalse(marker.exists()) + finally: + if target_pidfd is not None: + if not trace._linux_pidfd_has_exited(target_pidfd): + trace._linux_signal_pidfd(target_pidfd, trace.signal.SIGKILL) + os.close(target_pidfd) + if supervisor.poll() is None: + supervisor.kill() + supervisor.wait(timeout=5) + listener.close() + def test_linux_native_helper_stream_close_reports_every_failure(self) -> None: protocol = mock.Mock() protocol.close.side_effect = OSError("protocol") diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 09aa9ebf8c91..78be1cdc0c1d 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the unprivileged execution identity, makes mount propagation private, removes the inherited host procfs view, and mounts a procfs view owned by the new PID namespace. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. The namespace init binds its lifetime to the helper, executes the target only after release, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, clone3, pidfds, parent-death binding, mapping, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, removes the inherited host procfs view, and mounts a procfs view owned by the new PID namespace. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, empties supplemental groups, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, and isolated session before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp index 20d6ade1aa5e..2c3bb36b0385 100644 --- a/tools/deepseek-v41-trace/linux-containment-helper.cpp +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -3,6 +3,7 @@ #endif #include +#include #include #include #include @@ -16,14 +17,23 @@ #if defined(__linux__) #include +#include +#include +#include +#include #include +#include +#include #include #include #include +#include #include +#include #include #include #include +#include #include #include @@ -36,6 +46,15 @@ #if !defined(SYS_pidfd_open) #define SYS_pidfd_open 434 #endif +#if !defined(SYS_seccomp) +#define SYS_seccomp 317 +#endif +#if !defined(SECCOMP_RET_KILL_PROCESS) +#define SECCOMP_RET_KILL_PROCESS SECCOMP_RET_KILL +#endif +#if !defined(PR_SET_PTRACER) +#define PR_SET_PTRACER 0x59616d61 +#endif extern char ** environ; #endif @@ -188,6 +207,14 @@ void set_parent_death(pid_t expected_parent) { } } +void require_parent_death(pid_t expected_parent) { + int parent_signal = 0; + if (prctl(PR_GET_PDEATHSIG, &parent_signal) != 0 || + parent_signal != SIGKILL || getppid() != expected_parent) { + throw std::runtime_error("containment parent-death binding changed"); + } +} + void write_all(int fd, const void * data, size_t size) { const char * cursor = static_cast(data); while (size > 0) { @@ -228,9 +255,9 @@ void send_packet(int fd, const char * packet) { } } -void send_pidfd(int fd, int pidfd) { - char payload[] = "PREPARED"; - iovec vector {payload, sizeof(payload) - 1}; +void send_descriptor(int fd, const char * packet, int descriptor) { + const size_t packet_size = std::strlen(packet); + iovec vector {const_cast(packet), packet_size}; alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int))] {}; msghdr message {}; message.msg_iov = &vector; @@ -241,12 +268,46 @@ void send_pidfd(int fd, int pidfd) { header->cmsg_level = SOL_SOCKET; header->cmsg_type = SCM_RIGHTS; header->cmsg_len = CMSG_LEN(sizeof(int)); - std::memcpy(CMSG_DATA(header), &pidfd, sizeof(pidfd)); - if (sendmsg(fd, &message, MSG_NOSIGNAL) != static_cast(sizeof(payload) - 1)) { - throw std::runtime_error("cannot send namespace pidfd"); + std::memcpy(CMSG_DATA(header), &descriptor, sizeof(descriptor)); + if (sendmsg(fd, &message, MSG_NOSIGNAL) != static_cast(packet_size)) { + throw std::runtime_error("cannot send containment descriptor"); } } +int receive_descriptor(int fd, const char * expected_packet) { + char payload[32] {}; + iovec vector {payload, sizeof(payload)}; + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int))] {}; + msghdr message {}; + message.msg_iov = &vector; + message.msg_iovlen = 1; + message.msg_control = control; + message.msg_controllen = sizeof(control); + const ssize_t size = recvmsg(fd, &message, 0); + if (size != static_cast(std::strlen(expected_packet)) || + std::memcmp(payload, expected_packet, static_cast(size)) != 0 || + (message.msg_flags & (MSG_CTRUNC | MSG_TRUNC)) != 0) { + throw std::runtime_error("containment descriptor packet is invalid"); + } + cmsghdr * header = CMSG_FIRSTHDR(&message); + if (header == nullptr || + header->cmsg_level != SOL_SOCKET || + header->cmsg_type != SCM_RIGHTS || + header->cmsg_len != CMSG_LEN(sizeof(int)) || + CMSG_NXTHDR(&message, header) != nullptr) { + throw std::runtime_error("containment descriptor rights are invalid"); + } + int descriptor = -1; + std::memcpy(&descriptor, CMSG_DATA(header), sizeof(descriptor)); + if (descriptor < 0 || fcntl(descriptor, F_SETFD, FD_CLOEXEC) != 0) { + if (descriptor >= 0) { + close(descriptor); + } + throw std::runtime_error("cannot retain containment descriptor"); + } + return descriptor; +} + bool pidfd_has_exited(int pidfd) { pollfd descriptor {pidfd, POLLIN, 0}; const int result = poll(&descriptor, 1, 0); @@ -266,6 +327,389 @@ int wait_status_exit_code(int status) { return 125; } +void make_isolated_session() { + if (setsid() < 0 || getsid(0) != getpid() || getpgrp() != getpid()) { + throw std::runtime_error("cannot isolate containment session"); + } +} + +void protect_namespace_init() { + make_isolated_session(); + if (prctl(PR_SET_DUMPABLE, 0) != 0 || + prctl(PR_SET_PTRACER, 0) != 0 || + prctl(PR_GET_DUMPABLE) != 0) { + throw std::runtime_error("cannot protect target namespace init"); + } + require_parent_death(0); +} + +int read_cap_last_cap() { + const int fd = open("/proc/sys/kernel/cap_last_cap", O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) { + throw std::runtime_error("cannot open kernel capability limit"); + } + char buffer[32] {}; + const ssize_t size = read(fd, buffer, sizeof(buffer) - 1); + const int saved_errno = errno; + close_checked(fd, "kernel capability limit descriptor"); + if (size <= 0) { + errno = saved_errno; + throw std::runtime_error("cannot read kernel capability limit"); + } + char * end = nullptr; + errno = 0; + const long result = std::strtol(buffer, &end, 10); + if (errno != 0 || end == buffer || (*end != '\n' && *end != '\0') || + result < 0 || result > 63) { + throw std::runtime_error("kernel capability limit is invalid"); + } + return static_cast(result); +} + +uint32_t seccomp_audit_arch() { +#if defined(__x86_64__) + return AUDIT_ARCH_X86_64; +#elif defined(__aarch64__) + return AUDIT_ARCH_AARCH64; +#else +#error "Unsupported Linux architecture for DeepSeek V4.1 containment" +#endif +} + +void filter_statement(std::vector & filter, uint16_t code, uint32_t value) { + filter.push_back(sock_filter {code, 0, 0, value}); +} + +void filter_jump( + std::vector & filter, + uint16_t code, + uint32_t value, + uint8_t on_true, + uint8_t on_false) { + filter.push_back(sock_filter {code, on_true, on_false, value}); +} + +void filter_pid_argument( + std::vector & filter, + int syscall_number, + unsigned argument, + uint32_t denied_result) { + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, nr)); + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, static_cast(syscall_number), 0, 4); + filter_statement( + filter, + BPF_LD | BPF_W | BPF_ABS, + static_cast(offsetof(seccomp_data, args) + argument * sizeof(uint64_t))); + filter_jump(filter, BPF_JMP | BPF_JGE | BPF_K, 0x80000000U, 1, 0); + filter_jump(filter, BPF_JMP | BPF_JGT | BPF_K, 1, 1, 0); + filter_statement(filter, BPF_RET | BPF_K, denied_result); +} + +int install_target_seccomp_listener() { + std::vector denied_syscalls; +#define DSV41_DENY_SYSCALL(name) denied_syscalls.push_back(__NR_##name) +#if defined(__NR_ptrace) + DSV41_DENY_SYSCALL(ptrace); +#endif +#if defined(__NR_process_vm_readv) + DSV41_DENY_SYSCALL(process_vm_readv); +#endif +#if defined(__NR_process_vm_writev) + DSV41_DENY_SYSCALL(process_vm_writev); +#endif +#if defined(__NR_pidfd_send_signal) + DSV41_DENY_SYSCALL(pidfd_send_signal); +#endif +#if defined(__NR_setuid) + DSV41_DENY_SYSCALL(setuid); +#endif +#if defined(__NR_setgid) + DSV41_DENY_SYSCALL(setgid); +#endif +#if defined(__NR_setreuid) + DSV41_DENY_SYSCALL(setreuid); +#endif +#if defined(__NR_setregid) + DSV41_DENY_SYSCALL(setregid); +#endif +#if defined(__NR_setresuid) + DSV41_DENY_SYSCALL(setresuid); +#endif +#if defined(__NR_setresgid) + DSV41_DENY_SYSCALL(setresgid); +#endif +#if defined(__NR_setfsuid) + DSV41_DENY_SYSCALL(setfsuid); +#endif +#if defined(__NR_setfsgid) + DSV41_DENY_SYSCALL(setfsgid); +#endif +#if defined(__NR_setgroups) + DSV41_DENY_SYSCALL(setgroups); +#endif +#if defined(__NR_capset) + DSV41_DENY_SYSCALL(capset); +#endif +#if defined(__NR_setpgid) + DSV41_DENY_SYSCALL(setpgid); +#endif +#if defined(__NR_setsid) + DSV41_DENY_SYSCALL(setsid); +#endif +#if defined(__NR_setns) + DSV41_DENY_SYSCALL(setns); +#endif +#if defined(__NR_unshare) + DSV41_DENY_SYSCALL(unshare); +#endif +#undef DSV41_DENY_SYSCALL + + const int denied_prctl[] = { + PR_SET_PDEATHSIG, + PR_SET_DUMPABLE, + PR_SET_PTRACER, + PR_SET_SECUREBITS, + PR_SET_NO_NEW_PRIVS, + PR_CAPBSET_DROP, + PR_CAP_AMBIENT, + }; + constexpr uint32_t denied_result = SECCOMP_RET_USER_NOTIF; + std::vector filter; + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, arch)); + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, seccomp_audit_arch(), 1, 0); + filter_statement(filter, BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS); +#if defined(__NR_kill) + filter_pid_argument(filter, __NR_kill, 0, denied_result); +#endif +#if defined(__NR_tkill) + filter_pid_argument(filter, __NR_tkill, 0, denied_result); +#endif +#if defined(__NR_tgkill) + filter_pid_argument(filter, __NR_tgkill, 0, denied_result); + filter_pid_argument(filter, __NR_tgkill, 1, denied_result); +#endif +#if defined(__NR_rt_sigqueueinfo) + filter_pid_argument(filter, __NR_rt_sigqueueinfo, 0, denied_result); +#endif +#if defined(__NR_rt_tgsigqueueinfo) + filter_pid_argument(filter, __NR_rt_tgsigqueueinfo, 0, denied_result); + filter_pid_argument(filter, __NR_rt_tgsigqueueinfo, 1, denied_result); +#endif + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, nr)); + filter_jump( + filter, + BPF_JMP | BPF_JEQ | BPF_K, + __NR_prctl, + 0, + static_cast(2 + 2 * (sizeof(denied_prctl) / sizeof(denied_prctl[0])))); + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, args)); + for (int operation : denied_prctl) { + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, static_cast(operation), 0, 1); + filter_statement(filter, BPF_RET | BPF_K, denied_result); + } + filter_statement(filter, BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, nr)); + for (int syscall_number : denied_syscalls) { + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, static_cast(syscall_number), 0, 1); + filter_statement(filter, BPF_RET | BPF_K, denied_result); + } + filter_statement(filter, BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + if (filter.size() > std::numeric_limits::max()) { + throw std::runtime_error("target seccomp filter is too large"); + } + sock_fprog program { + static_cast(filter.size()), + filter.data(), + }; + const int listener = static_cast( + syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_NEW_LISTENER, &program)); + if (listener < 0 || fcntl(listener, F_SETFD, FD_CLOEXEC) != 0) { + if (listener >= 0) { + close(listener); + } + throw std::runtime_error("cannot install target seccomp filter"); + } + return listener; +} + +void require_eperm(long result, const char * label) { + if (result != -1 || errno != EPERM) { + throw std::runtime_error(std::string("target isolation did not deny ") + label); + } +} + +int drop_target_privileges() { + constexpr uid_t target_uid = 65534; + constexpr gid_t target_gid = 65534; + constexpr unsigned long securebits = + SECBIT_NOROOT | + SECBIT_NOROOT_LOCKED | + SECBIT_NO_SETUID_FIXUP | + SECBIT_NO_SETUID_FIXUP_LOCKED | + SECBIT_KEEP_CAPS_LOCKED | + SECBIT_NO_CAP_AMBIENT_RAISE | + SECBIT_NO_CAP_AMBIENT_RAISE_LOCKED; + const int cap_last_cap = read_cap_last_cap(); + if (prctl(PR_SET_SECUREBITS, securebits) != 0 || + setresgid(target_gid, target_gid, target_gid) != 0 || + setresuid(target_uid, target_uid, target_uid) != 0) { + throw std::runtime_error("cannot enter target non-root credentials"); + } + for (int capability = 0; capability <= cap_last_cap; ++capability) { + if (prctl(PR_CAPBSET_DROP, capability, 0, 0, 0) != 0) { + throw std::runtime_error("cannot drop target capability bounding set"); + } + } + if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0) != 0) { + throw std::runtime_error("cannot clear target ambient capabilities"); + } + __user_cap_header_struct header {}; + header.version = _LINUX_CAPABILITY_VERSION_3; + __user_cap_data_struct data[2] {}; + if (syscall(SYS_capset, &header, data) != 0 || + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + throw std::runtime_error("cannot clear target capabilities"); + } + __user_cap_data_struct verified[2] {}; + if (syscall(SYS_capget, &header, verified) != 0) { + throw std::runtime_error("cannot verify target capabilities"); + } + for (const __user_cap_data_struct & entry : verified) { + if (entry.effective != 0 || entry.permitted != 0 || entry.inheritable != 0) { + throw std::runtime_error("target capability sets are not empty"); + } + } + for (int capability = 0; capability <= cap_last_cap; ++capability) { + if (prctl(PR_CAPBSET_READ, capability, 0, 0, 0) != 0 || + prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_IS_SET, capability, 0, 0) != 0) { + throw std::runtime_error("target retained a bounded or ambient capability"); + } + } + if (getuid() != target_uid || geteuid() != target_uid || + getgid() != target_gid || getegid() != target_gid || + getgroups(0, nullptr) != 0 || + prctl(PR_GET_SECUREBITS) != static_cast(securebits) || + prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1 || + getsid(0) != getpid() || getpgrp() != getpid()) { + throw std::runtime_error("target privilege isolation verification failed"); + } + const int listener = install_target_seccomp_listener(); + if (prctl(PR_GET_SECCOMP, 0, 0, 0, 0) != SECCOMP_MODE_FILTER) { + close(listener); + throw std::runtime_error("target seccomp verification failed"); + } + return listener; +} + +void verify_target_attack_denials() { + errno = 0; + require_eperm(syscall(SYS_ptrace, PTRACE_ATTACH, 1, nullptr, nullptr), "ptrace"); +#if defined(SYS_process_vm_readv) + errno = 0; + require_eperm(syscall(SYS_process_vm_readv, 1, nullptr, 0, nullptr, 0, 0), "process_vm_readv"); +#endif +#if defined(SYS_process_vm_writev) + errno = 0; + require_eperm(syscall(SYS_process_vm_writev, 1, nullptr, 0, nullptr, 0, 0), "process_vm_writev"); +#endif + errno = 0; + require_eperm(kill(1, SIGSTOP), "namespace init signaling"); + errno = 0; + require_eperm(kill(-getpgrp(), 0), "process-group signaling"); + if (kill(getpid(), 0) != 0) { + throw std::runtime_error("target isolation blocked local signaling"); + } + errno = 0; + require_eperm(prctl(PR_SET_PDEATHSIG, 0), "parent-death mutation"); + errno = 0; + require_eperm(setresuid(0, 0, 0), "UID regain"); + errno = 0; + require_eperm(setpgid(0, 0), "process-group mutation"); + errno = 0; + require_eperm(setsid(), "session mutation"); + __user_cap_header_struct header {}; + header.version = _LINUX_CAPABILITY_VERSION_3; + __user_cap_data_struct data[2] {}; + errno = 0; + require_eperm(syscall(SYS_capset, &header, data), "capability regain"); +} + +struct seccomp_notification { + bool present; + uint64_t id; + seccomp_data data; +}; + +seccomp_notification receive_seccomp_notification(int listener, bool nonblocking) { + seccomp_notif_sizes sizes {}; + if (syscall(SYS_seccomp, SECCOMP_GET_NOTIF_SIZES, 0, &sizes) != 0 || + sizes.seccomp_notif < sizeof(seccomp_notif) || + sizes.seccomp_notif_resp < sizeof(seccomp_notif_resp)) { + throw std::runtime_error("cannot query target seccomp notification sizes"); + } + std::vector request_buffer( + (sizes.seccomp_notif + sizeof(uint64_t) - 1) / sizeof(uint64_t)); + seccomp_notif * request = reinterpret_cast(request_buffer.data()); + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, request) != 0) { + if (nonblocking && (errno == EAGAIN || errno == ENOENT)) { + return seccomp_notification {false, 0, {}}; + } + throw std::runtime_error("cannot receive target seccomp notification"); + } + if (request->flags != 0 || + ioctl(listener, SECCOMP_IOCTL_NOTIF_ID_VALID, &request->id) != 0) { + throw std::runtime_error("target seccomp notification identity is invalid"); + } + return seccomp_notification {true, request->id, request->data}; +} + +void deny_seccomp_notification(int listener, const seccomp_notification & notification) { + seccomp_notif_resp response {}; + response.id = notification.id; + response.error = -EPERM; + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &response) != 0) { + throw std::runtime_error("cannot deny target seccomp notification"); + } +} + +seccomp_notification expect_seccomp_notification( + int listener, + int syscall_number, + const char * label) { + const seccomp_notification notification = receive_seccomp_notification(listener, false); + if (!notification.present || notification.data.nr != syscall_number) { + throw std::runtime_error(std::string("unexpected target isolation probe: ") + label); + } + deny_seccomp_notification(listener, notification); + return notification; +} + +void verify_target_isolation_probes(int listener) { + expect_seccomp_notification(listener, SYS_ptrace, "ptrace"); +#if defined(SYS_process_vm_readv) + expect_seccomp_notification(listener, SYS_process_vm_readv, "process_vm_readv"); +#endif +#if defined(SYS_process_vm_writev) + expect_seccomp_notification(listener, SYS_process_vm_writev, "process_vm_writev"); +#endif + seccomp_notification notification = expect_seccomp_notification(listener, SYS_kill, "PID 1 signaling"); + if (notification.data.args[0] != 1 || notification.data.args[1] != SIGSTOP) { + throw std::runtime_error("target PID 1 signal probe is invalid"); + } + notification = expect_seccomp_notification(listener, SYS_kill, "process-group signaling"); + if (static_cast(notification.data.args[0]) >= 0) { + throw std::runtime_error("target process-group signal probe is invalid"); + } + notification = expect_seccomp_notification(listener, SYS_prctl, "parent-death mutation"); + if (notification.data.args[0] != PR_SET_PDEATHSIG) { + throw std::runtime_error("target parent-death probe is invalid"); + } + expect_seccomp_notification(listener, SYS_setresuid, "UID regain"); + expect_seccomp_notification(listener, SYS_setpgid, "process-group mutation"); + expect_seccomp_notification(listener, SYS_setsid, "session mutation"); + expect_seccomp_notification(listener, SYS_capset, "capability regain"); +} + class namespace_owner { public: namespace_owner(pid_t pid, int pidfd) : pid_(pid), pidfd_(pidfd) { @@ -314,6 +758,79 @@ class namespace_owner { bool reaped_ = false; }; +int wait_for_isolated_target(namespace_owner & target, int listener) { + const int flags = fcntl(listener, F_GETFL); + if (flags < 0 || fcntl(listener, F_SETFL, flags | O_NONBLOCK) != 0) { + throw std::runtime_error("cannot configure target seccomp listener"); + } + pollfd descriptors[2] { + {listener, POLLIN, 0}, + {target.pidfd(), POLLIN, 0}, + }; + while (true) { + const int result = poll(descriptors, 2, -1); + if (result < 0) { + if (errno == EINTR) { + continue; + } + throw std::runtime_error("cannot monitor isolated target"); + } + if ((descriptors[0].revents & (POLLIN | POLLHUP | POLLERR)) != 0) { + const seccomp_notification notification = receive_seccomp_notification(listener, true); + if (notification.present) { + throw std::runtime_error("target attempted a forbidden lifecycle operation"); + } + } + if ((descriptors[1].revents & (POLLIN | POLLHUP | POLLERR)) != 0) { + const seccomp_notification notification = receive_seccomp_notification(listener, true); + if (notification.present) { + throw std::runtime_error("target attempted a forbidden lifecycle operation"); + } + return target.wait(); + } + } +} + +[[noreturn]] void run_target_bootstrap( + int mapping_fd, + int ready_fd, + int security_fd, + int namespace_ready_fd, + const options & config) { + try { + set_parent_death(1); + make_isolated_session(); + close_checked(namespace_ready_fd, "target namespace readiness descriptor"); + write_all(ready_fd, "B", 1); + char mapped = 0; + ssize_t mapped_size; + do { + mapped_size = read(mapping_fd, &mapped, 1); + } while (mapped_size < 0 && errno == EINTR); + if (mapped_size != 1 || mapped != 'M') { + _exit(125); + } + close_checked(mapping_fd, "target mapping descriptor"); + const int listener = drop_target_privileges(); + send_descriptor(security_fd, "FILTER", listener); + close_checked(listener, "target seccomp listener"); + verify_target_attack_denials(); + send_packet(security_fd, "VERIFIED"); + if (receive_packet(security_fd) != "GO") { + _exit(125); + } + close_checked(security_fd, "target security descriptor"); + require_parent_death(1); + close_unneeded_fds(config.keep_fds, ready_fd); + write_all(ready_fd, "I", 1); + close_checked(ready_fd, "target readiness descriptor"); + execve(config.exec_path.c_str(), config.target_argv.data(), environ); + _exit(127); + } catch (...) { + _exit(125); + } +} + [[noreturn]] void run_namespace_init( int release_fd, int ready_fd, @@ -348,8 +865,8 @@ class namespace_owner { mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr) != 0) { _exit(125); } + protect_namespace_init(); write_all(ready_fd, "R", 1); - close_checked(ready_fd, "namespace readiness descriptor"); char release = 0; ssize_t received; do { @@ -359,36 +876,100 @@ class namespace_owner { _exit(125); } close_checked(release_fd, "namespace release descriptor"); - const pid_t target_pid = fork(); + if (setgroups(0, nullptr) != 0) { + _exit(125); + } + int target_mapping_pipe[2] {-1, -1}; + int target_ready_pipe[2] {-1, -1}; + int target_security[2] {-1, -1}; + if (pipe2(target_mapping_pipe, O_CLOEXEC) != 0 || + pipe2(target_ready_pipe, O_CLOEXEC) != 0 || + socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, target_security) != 0) { + _exit(125); + } + int target_pidfd = -1; + clone_args target_arguments {}; + target_arguments.flags = CLONE_NEWUSER | CLONE_PIDFD; + target_arguments.pidfd = reinterpret_cast(&target_pidfd); + target_arguments.exit_signal = SIGCHLD; + const pid_t target_pid = static_cast( + syscall(SYS_clone3, &target_arguments, sizeof(target_arguments))); if (target_pid < 0) { _exit(125); } if (target_pid == 0) { - for (int signal_number = 1; signal_number < NSIG; ++signal_number) { - if (signal_number == SIGKILL || signal_number == SIGSTOP) { - continue; - } - struct sigaction action {}; - action.sa_handler = SIG_DFL; - sigemptyset(&action.sa_mask); - sigaction(signal_number, &action, nullptr); + close_checked(target_mapping_pipe[1], "target mapping writer"); + close_checked(target_ready_pipe[0], "target readiness reader"); + close_checked(target_security[0], "target security supervisor descriptor"); + run_target_bootstrap( + target_mapping_pipe[0], target_ready_pipe[1], + target_security[1], ready_fd, config); + } + namespace_owner owned_target(target_pid, target_pidfd); + close_checked(target_mapping_pipe[0], "namespace target mapping reader"); + close_checked(target_ready_pipe[1], "namespace target readiness writer"); + close_checked(target_security[1], "namespace target security descriptor"); + char target_ready = 0; + ssize_t target_ready_size; + do { + target_ready_size = read(target_ready_pipe[0], &target_ready, 1); + } while (target_ready_size < 0 && errno == EINTR); + if (target_ready_size != 1 || target_ready != 'B') { + _exit(125); + } + const std::string target_process_path = "/proc/" + std::to_string(target_pid); + const int target_process_directory = open( + target_process_path.c_str(), O_PATH | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); + if (target_process_directory < 0 || pidfd_has_exited(target_pidfd)) { + _exit(125); + } + write_mapping_file(target_process_directory, "setgroups", "deny\n"); + write_mapping_file(target_process_directory, "uid_map", "65534 0 1\n"); + write_mapping_file(target_process_directory, "gid_map", "65534 0 1\n"); + close_checked(target_process_directory, "target process directory"); + write_all(target_mapping_pipe[1], "M", 1); + close_checked(target_mapping_pipe[1], "namespace target mapping writer"); + const int target_listener = receive_descriptor(target_security[0], "FILTER"); + verify_target_isolation_probes(target_listener); + if (receive_packet(target_security[0]) != "VERIFIED") { + _exit(125); + } + require_parent_death(0); + if (prctl(PR_GET_DUMPABLE) != 0 || + getsid(0) != getpid() || getpgrp() != getpid()) { + _exit(125); + } + send_packet(target_security[0], "GO"); + close_checked(target_security[0], "namespace target security supervisor descriptor"); + do { + target_ready_size = read(target_ready_pipe[0], &target_ready, 1); + } while (target_ready_size < 0 && errno == EINTR); + close_checked(target_ready_pipe[0], "namespace target readiness reader"); + require_parent_death(0); + if (prctl(PR_GET_DUMPABLE) != 0 || + getsid(0) != getpid() || getpgrp() != getpid() || + target_ready_size != 1 || target_ready != 'I') { + _exit(125); + } + write_all(ready_fd, "I", 1); + const int target_status = wait_for_isolated_target(owned_target, target_listener); + close_checked(target_listener, "namespace target seccomp listener"); + owned_target.close_pidfd(); + if (kill(-1, SIGKILL) != 0 && errno != ESRCH) { + _exit(125); + } + while (true) { + const pid_t reaped = waitpid(-1, nullptr, 0); + if (reaped > 0 || (reaped < 0 && errno == EINTR)) { + continue; } - sigset_t empty; - sigemptyset(&empty); - sigprocmask(SIG_SETMASK, &empty, nullptr); - close_unneeded_fds(config.keep_fds, -1); - execve(config.exec_path.c_str(), config.target_argv.data(), environ); - _exit(127); - } - int target_status = 0; - while (waitpid(target_pid, &target_status, 0) < 0) { - if (errno != EINTR) { - _exit(125); + if (reaped < 0 && errno == ECHILD) { + break; } + _exit(125); } - kill(-1, SIGKILL); - while (waitpid(-1, nullptr, 0) >= 0 || errno == EINTR) { - } + write_all(ready_fd, "C", 1); + close_checked(ready_fd, "namespace readiness descriptor"); _exit(wait_status_exit_code(target_status)); } catch (...) { _exit(125); @@ -492,19 +1073,31 @@ int run_linux_helper(options config) { do { ready_size = read(ready_pipe[0], &ready, 1); } while (ready_size < 0 && errno == EINTR); - close_checked(ready_pipe[0], "helper readiness reader"); close_checked(helper_pidfd, "helper self pidfd"); if (ready_size != 1 || ready != 'R') { throw std::runtime_error("target PID namespace setup did not complete"); } - send_pidfd(config.protocol_fd, owned_namespace.pidfd()); + send_descriptor(config.protocol_fd, "PREPARED", owned_namespace.pidfd()); if (receive_packet(config.protocol_fd) != "EXEC") { throw std::runtime_error("containment EXEC packet is invalid"); } write_all(release_pipe[1], "X", 1); close_checked(release_pipe[1], "helper release writer"); + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + if (ready_size != 1 || ready != 'I') { + throw std::runtime_error("target privilege isolation did not complete"); + } send_packet(config.protocol_fd, "RELEASED"); const int status = owned_namespace.wait(); + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + close_checked(ready_pipe[0], "helper readiness reader"); + if (ready_size != 1 || ready != 'C') { + throw std::runtime_error("target namespace teardown did not complete"); + } owned_namespace.close_pidfd(); send_packet(config.protocol_fd, "COMPLETE"); return wait_status_exit_code(status); diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index ea6690d60810..5fe289216b5b 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -1216,6 +1216,7 @@ def _start_linux_native_helper_process( helper_argv, os.environ if environment is None else environment, file_actions=file_actions, + setsid=True, setsigmask=_all_catchable_signals(), setsigdef=_all_catchable_signals(), ) From bf3d930827016bd798238a264cde7e4435d28ba7 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 19:10:34 -0700 Subject: [PATCH 44/56] trace : diagnose Linux containment startup Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 462 ++++++++++++++++-- tools/deepseek-v41-trace/README.md | 2 +- .../linux-containment-helper.cpp | 405 +++++++++++++-- tools/deepseek-v41-trace/trace_format.py | 47 +- 4 files changed, 831 insertions(+), 85 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 6b2029260a75..7d0f7c81543e 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import array import contextlib import copy import importlib.util @@ -45,6 +46,96 @@ TEST_CANDIDATE_EXPORTER_POLICY_ID = "test-candidate-exporter" TEST_DS4_EXPORTER_POLICY_ID = "test-ds4-exporter" TEST_PROMPT_BUILDER_POLICY_ID = "test-prompt-builder" +NATIVE_TEST_STATE_PACKET_MAX_BYTES = 4096 +NATIVE_TEST_STATE_KEYS = frozenset({ + "caps", + "gids", + "groups", + "no_new_privs", + "pgrp", + "pid", + "seccomp", + "securebits", + "sid", + "uids", +}) + + +def receive_native_test_pidfd_packet(connection, max_payload_bytes): + import array + import socket + + item_size = array.array("i").itemsize + requested_flags = getattr(socket, "MSG_CMSG_CLOEXEC", 0) + data, ancillary, flags, _address = connection.recvmsg( + max_payload_bytes + 1, + socket.CMSG_SPACE(item_size), + requested_flags, + ) + descriptors = [] + try: + for level, kind, content in ancillary: + if level != socket.SOL_SOCKET or kind != socket.SCM_RIGHTS: + raise AssertionError("native test packet contained unexpected ancillary data") + descriptor_bytes = array.array("i") + descriptor_bytes.frombytes(content[:len(content) - len(content) % item_size]) + descriptors.extend(descriptor_bytes) + truncation_flags = ( + getattr(socket, "MSG_TRUNC", 0) | + getattr(socket, "MSG_CTRUNC", 0) + ) + if flags & truncation_flags: + raise AssertionError("native test packet was truncated") + if flags not in {0, requested_flags}: + raise AssertionError(f"native test packet returned unexpected flags {flags}") + if len(data) > max_payload_bytes: + raise AssertionError("native test packet exceeded its bounded payload") + if len(descriptors) != 1: + raise AssertionError("native test packet did not provide exactly one pidfd") + return data, descriptors.pop() + finally: + for descriptor in descriptors: + os.close(descriptor) + + +def decode_native_test_report(data): + if len(data) > NATIVE_TEST_STATE_PACKET_MAX_BYTES: + raise AssertionError("native test report exceeded its bounded payload") + try: + decoded = data.decode("ascii") + except UnicodeDecodeError as error: + raise AssertionError("native test report was not ASCII") from error + label, separator, payload = decoded.partition(":") + if label == "descendant": + if separator or payload: + raise AssertionError("descendant report contained an unexpected payload") + return label, None + if label != "target" or not separator or not payload: + raise AssertionError("native test report had an invalid label or payload") + try: + state = json.loads(payload) + except json.JSONDecodeError as error: + raise AssertionError("native target state was not valid JSON") from error + if not isinstance(state, dict) or set(state) != NATIVE_TEST_STATE_KEYS: + raise AssertionError("native target state schema did not match") + return label, state + + +def finish_native_test_supervisor(supervisor, *, kill_if_running): + try: + if kill_if_running and supervisor.poll() is None: + supervisor.kill() + _stdout, stderr = supervisor.communicate(timeout=5) + return supervisor.returncode, stderr or "" + except subprocess.TimeoutExpired: + if supervisor.poll() is None: + supervisor.kill() + supervisor.communicate(timeout=5) + raise + finally: + for stream in (supervisor.stdin, supervisor.stdout, supervisor.stderr): + if stream is not None and not stream.closed: + stream.close() WATCHDOG_EVENTS = [ { @@ -2771,6 +2862,192 @@ def test_linux_namespace_unavailable_never_sends_exec(self) -> None: self.assertEqual(protocol.sendall.call_args_list, [mock.call(b"PREPARE")]) self.assertIsNone(process.namespace_pidfd) + def test_linux_protocol_accepts_echoed_cmsg_cloexec(self) -> None: + protocol = mock.Mock() + protocol.recvmsg.return_value = (b"READY", [], 1073741824, None) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True): + process._receive_protocol(b"READY") + protocol.recvmsg.assert_called_once_with( + 128, + trace.socket.CMSG_SPACE(array.array("i").itemsize), + 1073741824, + ) + + def test_linux_protocol_rejects_truncation_and_unknown_flags(self) -> None: + for flags in (1073741824 | 32, 1073741824 | 8, 536870912): + with self.subTest(flags=flags): + protocol = mock.Mock() + protocol.recvmsg.return_value = (b"READY", [], flags, None) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), self.assertRaisesRegex( + trace.TraceError, "protocol expected READY"): + process._receive_protocol(b"READY") + + def test_native_pidfd_packet_rejects_truncation_unknown_flags_and_oversize(self) -> None: + import socket + + item_size = array.array("i").itemsize + rights = array.array("i", [71]).tobytes() + cases = ( + ((b"ready", [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], 1073741824 | 32, None), 5), + ((b"ready", [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], 536870912, None), 5), + ((b"ready!", [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], 1073741824, None), 5), + ) + for packet, bound in cases: + with self.subTest(flags=packet[2], size=len(packet[0])): + connection = mock.Mock() + connection.recvmsg.return_value = packet + with mock.patch( + "socket.MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + os, "close") as close, self.assertRaises(AssertionError): + receive_native_test_pidfd_packet(connection, bound) + close.assert_called_once_with(71) + connection.recvmsg.assert_called_once_with( + bound + 1, + socket.CMSG_SPACE(item_size), + 1073741824, + ) + + def test_native_pidfd_packet_requires_one_descriptor(self) -> None: + import socket + + rights = array.array("i", [71, 72]).tobytes() + connection = mock.Mock() + connection.recvmsg.return_value = ( + b"ready", + [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], + 0, + None, + ) + with mock.patch.object(os, "close") as close, self.assertRaisesRegex( + AssertionError, "exactly one pidfd"): + receive_native_test_pidfd_packet(connection, len(b"ready")) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_startup_failure_reaps_and_closes_streams(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=84, + ) + process.root_pidfd = 90 + primary = trace.TraceError("protocol startup failed") + cleanup = trace._ContainmentCleanup([], True) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( + process, "release_exec", side_effect=primary), mock.patch.object( + process, "abort_blocked", return_value=cleanup) as abort, mock.patch.object( + process, "collect_startup_stderr", + return_value=(b"stage=namespace-mount-proc errno=1\n", [])) as collect_stderr, mock.patch.object( + process, "close_streams", return_value=[]) as close_streams, mock.patch.object( + trace.os, "close"), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper(["approved"], {}) + self.assertIs(raised.exception.primary_error, primary) + self.assertFalse(raised.exception.quiescence_proven) + self.assertIn( + "helper stderr b'stage=namespace-mount-proc errno=1\\n'", + str(raised.exception), + ) + abort.assert_called_once() + collect_stderr.assert_called_once() + close_streams.assert_called_once() + lock.release.assert_called_once() + + def test_linux_startup_stderr_capture_is_bounded_and_closed(self) -> None: + read_fd, write_fd = os.pipe() + os.write(write_fd, b"exact helper diagnostic\n") + os.close(write_fd) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=read_fd, + ) + stderr, failures = process.collect_startup_stderr() + self.assertEqual(stderr, b"exact helper diagnostic\n") + self.assertEqual(failures, []) + self.assertIsNone(process._stderr_fd) + with self.assertRaises(OSError): + os.fstat(read_fd) + + process._stderr_fd = 84 + oversized = b"x" * (trace.PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES + 1) + with mock.patch.object(trace.os, "read", side_effect=[oversized]), mock.patch.object( + trace.os, "close") as close: + stderr, failures = process.collect_startup_stderr() + self.assertEqual(len(stderr), trace.PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES) + self.assertEqual( + [failure.component for failure in failures], + ["linux-helper-stderr-bounds"], + ) + close.assert_called_once_with(84) + + def test_native_test_report_is_bounded_and_schema_exact(self) -> None: + state = { + "uids": [65534] * 3, + "gids": [65534] * 3, + "groups": [], + "sid": 3, + "pgrp": 3, + "pid": 3, + "caps": ["0000000000000000"] * 5, + "no_new_privs": "1", + "seccomp": "2", + "securebits": 239, + } + encoded = ( + "target:" + json.dumps(state, sort_keys=True, separators=(",", ":")) + ).encode("ascii") + self.assertEqual(decode_native_test_report(encoded), ("target", state)) + self.assertEqual(decode_native_test_report(b"descendant"), ("descendant", None)) + for invalid in ( + b"x" * (NATIVE_TEST_STATE_PACKET_MAX_BYTES + 1), + b"target:{", + b"target:{}", + b"descendant:payload"): + with self.subTest(invalid=invalid[:32]), self.assertRaises(AssertionError): + decode_native_test_report(invalid) + + def test_native_supervisor_stderr_is_preserved_and_closed(self) -> None: + supervisor = subprocess.Popen( + [sys.executable, "-c", "import sys;sys.stderr.write('helper failure')"], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + self.assertEqual(returncode, 0) + self.assertEqual(stderr, "helper failure") + self.assertTrue(supervisor.stderr.closed) + def test_linux_namespace_authority_precedes_release_completion(self) -> None: events = [] lock = mock.Mock() @@ -2824,6 +3101,115 @@ def test_linux_native_helper_source_binds_supervisor_death_and_namespace_lifecyc self.assertIn("require_parent_death(0);", helper_source) self.assertIn("require_parent_death(1);", helper_source) + def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + diagnostic_source = helper_source[ + helper_source.index("constexpr uint32_t DIAGNOSTIC_MAGIC"): + helper_source.index("bool retained_fd(") + ] + self.assertIn("constexpr size_t DIAGNOSTIC_STAGE_CAPACITY = 48;", diagnostic_source) + self.assertIn("static_assert(sizeof(failure_diagnostic) <= PIPE_BUF);", diagnostic_source) + self.assertIn("record.magic != DIAGNOSTIC_MAGIC", diagnostic_source) + self.assertIn("record.version != DIAGNOSTIC_VERSION", diagnostic_source) + self.assertIn("record.stage_size > DIAGNOSTIC_STAGE_CAPACITY", diagnostic_source) + self.assertIn("invalid setup diagnostic", diagnostic_source) + self.assertIn("count < 8", diagnostic_source) + self.assertIn("pipe2(diagnostic_pipe, O_CLOEXEC | O_NONBLOCK)", helper_source) + self.assertIn( + "run_namespace_init(\n" + " release_pipe[0], ready_pipe[1], mapping_pipe[0], diagnostic_pipe[1],", + helper_source, + ) + self.assertGreaterEqual( + helper_source.count("throw_setup_failure("), + 5, + ) + required_stages = { + "namespace-parent-death", + "namespace-parent-identity", + "namespace-mapping-read", + "namespace-setresgid", + "namespace-setresuid", + "namespace-mount-private", + "namespace-unmount-proc", + "namespace-mount-proc", + "namespace-verify-proc", + "namespace-protect-init", + "namespace-ready", + "target-parent-death", + "target-session", + "target-mapping-read", + "target-privilege-drop", + "target-isolation-probes", + "target-isolation-ready", + "target-exec", + } + stages = [ + line.split('stage = "', 1)[1].split('"', 1)[0] + for line in helper_source.splitlines() + if 'stage = "' in line + ] + self.assertEqual(len(stages), len(set(stages))) + self.assertTrue(all(0 < len(stage) <= 48 for stage in stages)) + self.assertTrue(all( + stage.startswith(("namespace-", "target-")) + for stage in stages + )) + for stage in required_stages: + self.assertIn(f'"{stage}"', helper_source) + child_source = helper_source[ + helper_source.index("[[noreturn]] void run_target_bootstrap"): + helper_source.index("int run_linux_helper") + ] + self.assertNotIn("_exit(125)", child_source) + self.assertIn("fail_stage(diagnostic_fd, stage, errno);", child_source) + + def test_linux_native_tests_do_not_hide_startup_failures(self) -> None: + source = ( + inspect.getsource(self.test_linux_native_helper_parent_death_boundary) + + inspect.getsource(self.test_linux_native_helper_forbidden_operations_kill_namespace) + ) + self.assertNotIn("skipTest(", source) + self.assertNotIn("'stderr':-3", source) + self.assertGreaterEqual(source.count("finish_native_test_supervisor("), 4) + self.assertGreaterEqual(source.count("self.fail("), 2) + + def test_linux_native_helper_procfs_overmount_is_verified_and_fail_closed(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + setup_source = helper_source[ + helper_source.index('stage = "namespace-mount-private"'): + helper_source.index('stage = "namespace-protect-init"') + ] + self.assertIn( + 'umount2("/proc", MNT_DETACH) != 0 && errno != EINVAL', + setup_source, + ) + self.assertIn( + 'mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr)', + setup_source, + ) + self.assertLess( + setup_source.index('stage = "namespace-unmount-proc"'), + setup_source.index('stage = "namespace-mount-proc"'), + ) + self.assertLess( + setup_source.index('stage = "namespace-mount-proc"'), + setup_source.index("verify_private_procfs();"), + ) + verification_source = helper_source[ + helper_source.index("void verify_private_procfs()"): + helper_source.index("void require_initial_signal_state()") + ] + self.assertIn("PROC_SUPER_MAGIC", verification_source) + self.assertIn('readlink("/proc/self"', verification_source) + self.assertIn("size != 1 || self_target[0] != '1'", verification_source) + @unittest.skipUnless( sys.platform == "linux" and os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER"), "native Linux containment helper was not executed on this host", @@ -2875,7 +3261,7 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: "'pass_fds':(target,)," "'_containment_helper_path':f'/proc/self/fd/{helper_fd}'," "'_containment_helper_descriptor':helper_fd," - "'stdout':-3,'stderr':-3}\n" + "'stdout':-3}\n" "containment=trace._start_linux_native_helper(" "[sys.executable,'-c',sys.argv[2],sys.argv[3],str(os.getpgrp())],launch)\n" "while True: time.sleep(1)\n" @@ -2892,6 +3278,7 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: text=True, ) pidfds = [] + supervisor_reaped = False try: labels = set() target_state = None @@ -2900,25 +3287,23 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: connection, _address = listener.accept() except TimeoutError: if supervisor.poll() is not None: - stderr = supervisor.stderr.read() - self.skipTest(f"native PID namespace unavailable: {stderr.strip()}") + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + supervisor_reaped = True + self.fail( + f"native helper failed before target report " + f"(exit {returncode}); stderr={stderr!r}") raise with connection: - descriptors = array.array("i") - data, ancillary, flags, _address = connection.recvmsg( - 32, socket.CMSG_SPACE(descriptors.itemsize)) - self.assertEqual(flags, 0) - for level, kind, content in ancillary: - if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: - descriptors.frombytes(content[:descriptors.itemsize]) - self.assertEqual(len(descriptors), 1) - decoded = data.decode("ascii") - label, separator, payload = decoded.partition(":") + data, pidfd = receive_native_test_pidfd_packet( + connection, NATIVE_TEST_STATE_PACKET_MAX_BYTES) + pidfds.append(pidfd) + label, state = decode_native_test_report(data) labels.add(label) - if separator: - target_state = json.loads(payload) - pidfds.append(descriptors[0]) + if state is not None: + target_state = state self.assertEqual(labels, {"target", "descendant"}) + self.assertIsNotNone(target_state) self.assertEqual(target_state["uids"], [65534] * 3) self.assertEqual(target_state["gids"], [65534] * 3) self.assertEqual(target_state["groups"], []) @@ -2928,8 +3313,10 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: self.assertEqual(target_state["no_new_privs"], "1") self.assertEqual(target_state["seccomp"], "2") self.assertEqual(target_state["securebits"], 239) - supervisor.kill() - supervisor.wait(timeout=5) + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=True) + supervisor_reaped = True + self.assertEqual(returncode, -trace.signal.SIGKILL, stderr) deadline = time.monotonic() + 5 while time.monotonic() < deadline and not all( trace._linux_pidfd_has_exited(pidfd) for pidfd in pidfds): @@ -2941,9 +3328,8 @@ def test_linux_native_helper_parent_death_boundary(self) -> None: if not trace._linux_pidfd_has_exited(pidfd): trace._linux_signal_pidfd(pidfd, trace.signal.SIGKILL) os.close(pidfd) - if supervisor.poll() is None: - supervisor.kill() - supervisor.wait(timeout=5) + if not supervisor_reaped: + finish_native_test_supervisor(supervisor, kill_if_running=True) listener.close() @unittest.skipUnless( @@ -2990,7 +3376,7 @@ def test_linux_native_helper_forbidden_operations_kill_namespace(self) -> None: "'pass_fds':(target,)," "'_containment_helper_path':f'/proc/self/fd/{helper_fd}'," "'_containment_helper_descriptor':helper_fd," - "'stdout':-3,'stderr':-3}\n" + "'stdout':-3}\n" "containment=trace._start_linux_native_helper(" "[sys.executable,'-c',sys.argv[2],sys.argv[3],sys.argv[4]," "str(os.getpgrp()),sys.argv[5]],launch)\n" @@ -3040,26 +3426,27 @@ def test_linux_native_helper_forbidden_operations_kill_namespace(self) -> None: text=True, ) target_pidfd = None + supervisor_reaped = False try: try: connection, _address = listener.accept() except TimeoutError: if supervisor.poll() is not None: - stderr = supervisor.stderr.read() - self.skipTest( - f"native PID namespace unavailable: {stderr.strip()}") + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + supervisor_reaped = True + self.fail( + f"native helper failed before attack report " + f"(exit {returncode}); stderr={stderr!r}") raise with connection: - descriptors = array.array("i") - data, ancillary, flags, _address = connection.recvmsg( - 32, socket.CMSG_SPACE(descriptors.itemsize)) - self.assertEqual((data, flags), (b"ready", 0)) - for level, kind, content in ancillary: - if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: - descriptors.frombytes(content[:descriptors.itemsize]) - self.assertEqual(len(descriptors), 1) - target_pidfd = descriptors[0] - self.assertEqual(supervisor.wait(timeout=10), 125) + data, target_pidfd = receive_native_test_pidfd_packet( + connection, len(b"ready")) + self.assertEqual(data, b"ready") + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + supervisor_reaped = True + self.assertEqual(returncode, 125, stderr) deadline = time.monotonic() + 5 while time.monotonic() < deadline and not trace._linux_pidfd_has_exited( target_pidfd): @@ -3071,9 +3458,8 @@ def test_linux_native_helper_forbidden_operations_kill_namespace(self) -> None: if not trace._linux_pidfd_has_exited(target_pidfd): trace._linux_signal_pidfd(target_pidfd, trace.signal.SIGKILL) os.close(target_pidfd) - if supervisor.poll() is None: - supervisor.kill() - supervisor.wait(timeout=5) + if not supervisor_reaped: + finish_native_test_supervisor(supervisor, kill_if_running=True) listener.close() def test_linux_native_helper_stream_close_reports_every_failure(self) -> None: diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 78be1cdc0c1d..0006c8b90206 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,7 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, removes the inherited host procfs view, and mounts a procfs view owned by the new PID namespace. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, empties supplemental groups, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, and isolated session before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, accepts only `EINVAL` when a locked inherited `/proc` cannot be detached, mounts a new `nosuid,nodev,noexec` procfs over `/proc`, and verifies `/proc/self` resolves to namespace PID 1. A separate bounded binary diagnostic pipe reports setup stage and errno without sharing protocol tokens or exposing host data. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, empties supplemental groups, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, and isolated session before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp index 2c3bb36b0385..d520906d21e4 100644 --- a/tools/deepseek-v41-trace/linux-containment-helper.cpp +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -3,6 +3,7 @@ #endif #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +36,7 @@ #include #include #include +#include #include #include @@ -121,11 +124,115 @@ options parse_options(int argc, char ** argv) { #if defined(__linux__) -bool retained_fd(int fd, const std::vector & keep_fds, int extra_fd) { +constexpr uint32_t DIAGNOSTIC_MAGIC = 0x44535634U; +constexpr uint16_t DIAGNOSTIC_VERSION = 1; +constexpr size_t DIAGNOSTIC_STAGE_CAPACITY = 48; + +struct failure_diagnostic { + uint32_t magic; + uint16_t version; + uint16_t stage_size; + int32_t error_number; + char stage[DIAGNOSTIC_STAGE_CAPACITY]; +}; + +static_assert(sizeof(failure_diagnostic) <= PIPE_BUF); + +void close_checked(int fd, const char * label); + +void report_failure(int fd, const char * stage, int error_number) noexcept { + const int saved_errno = errno; + failure_diagnostic record {}; + record.magic = DIAGNOSTIC_MAGIC; + record.version = DIAGNOSTIC_VERSION; + record.error_number = error_number; + while (record.stage_size < DIAGNOSTIC_STAGE_CAPACITY && + stage[record.stage_size] != '\0') { + record.stage[record.stage_size] = stage[record.stage_size]; + ++record.stage_size; + } + ssize_t written; + do { + written = write(fd, &record, sizeof(record)); + } while (written < 0 && errno == EINTR); + errno = saved_errno; +} + +[[noreturn]] void fail_stage( + int diagnostic_fd, + const char * stage, + int error_number, + int exit_code = 125) noexcept { + report_failure(diagnostic_fd, stage, error_number); + _exit(exit_code); +} + +std::string receive_failure_diagnostics(int fd) { + std::string result; + for (size_t count = 0; count < 8; ++count) { + failure_diagnostic record {}; + ssize_t size; + do { + size = read(fd, &record, sizeof(record)); + } while (size < 0 && errno == EINTR); + if (size < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + break; + } + if (size == 0) { + break; + } + if (size != static_cast(sizeof(record)) || + record.magic != DIAGNOSTIC_MAGIC || + record.version != DIAGNOSTIC_VERSION || + record.stage_size == 0 || + record.stage_size > DIAGNOSTIC_STAGE_CAPACITY) { + if (!result.empty()) { + result += "; "; + } + result += "invalid setup diagnostic"; + continue; + } + if (!result.empty()) { + result += "; "; + } + result += "stage="; + result.append(record.stage, record.stage_size); + result += " errno=" + std::to_string(record.error_number); + if (record.error_number != 0) { + result += " ("; + result += std::strerror(record.error_number); + result += ")"; + } + } + return result; +} + +[[noreturn]] void throw_setup_failure(int diagnostic_fd, const char * fallback) { + const std::string diagnostic = receive_failure_diagnostics(diagnostic_fd); + const int close_result = close(diagnostic_fd); + const int close_error = close_result == 0 ? 0 : errno; + std::string message = fallback; + if (!diagnostic.empty()) { + message += ": " + diagnostic; + } + if (close_error != 0) { + message += "; diagnostics-close errno=" + std::to_string(close_error); + message += " ("; + message += std::strerror(close_error); + message += ")"; + } + throw std::runtime_error(message); +} + +bool retained_fd( + int fd, + const std::vector & keep_fds, + int extra_fd, + int second_extra_fd = -1) { if (fd >= 0 && fd <= STDERR_FILENO) { return true; } - if (fd == extra_fd) { + if (fd == extra_fd || fd == second_extra_fd) { return true; } for (int keep_fd : keep_fds) { @@ -142,7 +249,10 @@ void close_checked(int fd, const char * label) { } } -void close_unneeded_fds(const std::vector & keep_fds, int extra_fd) { +void close_unneeded_fds( + const std::vector & keep_fds, + int extra_fd, + int second_extra_fd = -1) { DIR * directory = opendir("/proc/self/fd"); if (directory == nullptr) { throw std::runtime_error("cannot open procfs descriptor directory"); @@ -156,7 +266,8 @@ void close_unneeded_fds(const std::vector & keep_fds, int extra_fd) { continue; } const int fd = static_cast(parsed); - if (fd != directory_fd && !retained_fd(fd, keep_fds, extra_fd)) { + if (fd != directory_fd && + !retained_fd(fd, keep_fds, extra_fd, second_extra_fd)) { close_checked(fd, "unneeded descriptor"); } } @@ -165,6 +276,22 @@ void close_unneeded_fds(const std::vector & keep_fds, int extra_fd) { } } +void verify_private_procfs() { + struct statfs filesystem {}; + if (statfs("/proc", &filesystem) != 0) { + throw std::runtime_error("cannot inspect private procfs"); + } + if (static_cast(filesystem.f_type) != + static_cast(PROC_SUPER_MAGIC)) { + throw std::runtime_error("private procfs has an unexpected filesystem type"); + } + char self_target[32] {}; + const ssize_t size = readlink("/proc/self", self_target, sizeof(self_target)); + if (size != 1 || self_target[0] != '1') { + throw std::runtime_error("private procfs is not bound to the target PID namespace"); + } +} + void require_initial_signal_state() { sigset_t mask; if (sigprocmask(SIG_SETMASK, nullptr, &mask) != 0) { @@ -796,38 +923,74 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { int ready_fd, int security_fd, int namespace_ready_fd, + int diagnostic_fd, const options & config) { + const char * stage = "target-parent-death"; try { + errno = 0; set_parent_death(1); + stage = "target-session"; + errno = 0; make_isolated_session(); + stage = "target-namespace-ready-close"; + errno = 0; close_checked(namespace_ready_fd, "target namespace readiness descriptor"); + stage = "target-bound"; + errno = 0; write_all(ready_fd, "B", 1); + stage = "target-mapping-read"; char mapped = 0; ssize_t mapped_size; do { mapped_size = read(mapping_fd, &mapped, 1); } while (mapped_size < 0 && errno == EINTR); if (mapped_size != 1 || mapped != 'M') { - _exit(125); + fail_stage(diagnostic_fd, stage, mapped_size < 0 ? errno : 0); } + stage = "target-mapping-close"; + errno = 0; close_checked(mapping_fd, "target mapping descriptor"); + stage = "target-privilege-drop"; + errno = 0; const int listener = drop_target_privileges(); + stage = "target-filter-send"; + errno = 0; send_descriptor(security_fd, "FILTER", listener); + stage = "target-listener-close"; + errno = 0; close_checked(listener, "target seccomp listener"); + stage = "target-isolation-probes"; + errno = 0; verify_target_attack_denials(); + stage = "target-verified-send"; + errno = 0; send_packet(security_fd, "VERIFIED"); + stage = "target-go-read"; + errno = 0; if (receive_packet(security_fd) != "GO") { - _exit(125); + fail_stage(diagnostic_fd, stage, 0); } + stage = "target-security-close"; + errno = 0; close_checked(security_fd, "target security descriptor"); + stage = "target-parent-death-verify"; + errno = 0; require_parent_death(1); - close_unneeded_fds(config.keep_fds, ready_fd); + stage = "target-fd-close"; + errno = 0; + close_unneeded_fds(config.keep_fds, ready_fd, diagnostic_fd); + stage = "target-isolation-ready"; + errno = 0; write_all(ready_fd, "I", 1); + stage = "target-ready-close"; + errno = 0; close_checked(ready_fd, "target readiness descriptor"); + stage = "target-exec"; + errno = 0; execve(config.exec_path.c_str(), config.target_argv.data(), environ); - _exit(127); + fail_stage(diagnostic_fd, stage, errno, 127); } catch (...) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } } @@ -835,129 +998,238 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { int release_fd, int ready_fd, int mapping_fd, + int diagnostic_fd, int protocol_fd, int helper_pidfd, const options & config) { + const char * stage = "namespace-parent-death"; try { + errno = 0; if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } + stage = "namespace-parent-identity"; + errno = 0; if (getppid() != 0 || pidfd_has_exited(helper_pidfd)) { - _exit(125); + fail_stage(diagnostic_fd, stage, 0); } + stage = "namespace-helper-pidfd-close"; + errno = 0; close_checked(helper_pidfd, "namespace helper pidfd"); + stage = "namespace-protocol-close"; + errno = 0; close_checked(protocol_fd, "namespace protocol descriptor"); + stage = "namespace-bound"; + errno = 0; write_all(ready_fd, "B", 1); + stage = "namespace-mapping-read"; char mapped = 0; ssize_t mapped_size; do { mapped_size = read(mapping_fd, &mapped, 1); } while (mapped_size < 0 && errno == EINTR); if (mapped_size != 1 || mapped != 'M') { - _exit(125); + fail_stage(diagnostic_fd, stage, mapped_size < 0 ? errno : 0); } + stage = "namespace-mapping-close"; + errno = 0; close_checked(mapping_fd, "namespace mapping descriptor"); - if (setresgid(0, 0, 0) != 0 || setresuid(0, 0, 0) != 0) { - _exit(125); + stage = "namespace-setresgid"; + errno = 0; + if (setresgid(0, 0, 0) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-setresuid"; + errno = 0; + if (setresuid(0, 0, 0) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-mount-private"; + errno = 0; + if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0) { + fail_stage(diagnostic_fd, stage, errno); } - if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0 || - umount2("/proc", MNT_DETACH) != 0 || - mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr) != 0) { - _exit(125); + stage = "namespace-unmount-proc"; + errno = 0; + if (umount2("/proc", MNT_DETACH) != 0 && errno != EINVAL) { + fail_stage(diagnostic_fd, stage, errno); } + stage = "namespace-mount-proc"; + errno = 0; + if (mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-verify-proc"; + errno = 0; + verify_private_procfs(); + stage = "namespace-protect-init"; + errno = 0; protect_namespace_init(); + stage = "namespace-ready"; + errno = 0; write_all(ready_fd, "R", 1); + stage = "namespace-release-read"; char release = 0; ssize_t received; do { received = read(release_fd, &release, 1); } while (received < 0 && errno == EINTR); if (received != 1 || release != 'X') { - _exit(125); + fail_stage(diagnostic_fd, stage, received < 0 ? errno : 0); } + stage = "namespace-release-close"; + errno = 0; close_checked(release_fd, "namespace release descriptor"); + stage = "namespace-setgroups"; + errno = 0; if (setgroups(0, nullptr) != 0) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } int target_mapping_pipe[2] {-1, -1}; int target_ready_pipe[2] {-1, -1}; int target_security[2] {-1, -1}; - if (pipe2(target_mapping_pipe, O_CLOEXEC) != 0 || - pipe2(target_ready_pipe, O_CLOEXEC) != 0 || - socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, target_security) != 0) { - _exit(125); + stage = "namespace-target-mapping-pipe"; + errno = 0; + if (pipe2(target_mapping_pipe, O_CLOEXEC) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-target-ready-pipe"; + errno = 0; + if (pipe2(target_ready_pipe, O_CLOEXEC) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-target-security-socket"; + errno = 0; + if (socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, target_security) != 0) { + fail_stage(diagnostic_fd, stage, errno); } int target_pidfd = -1; clone_args target_arguments {}; target_arguments.flags = CLONE_NEWUSER | CLONE_PIDFD; target_arguments.pidfd = reinterpret_cast(&target_pidfd); target_arguments.exit_signal = SIGCHLD; + stage = "namespace-target-clone"; + errno = 0; const pid_t target_pid = static_cast( syscall(SYS_clone3, &target_arguments, sizeof(target_arguments))); if (target_pid < 0) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } if (target_pid == 0) { + stage = "target-handoff-close"; + errno = 0; close_checked(target_mapping_pipe[1], "target mapping writer"); close_checked(target_ready_pipe[0], "target readiness reader"); close_checked(target_security[0], "target security supervisor descriptor"); run_target_bootstrap( target_mapping_pipe[0], target_ready_pipe[1], - target_security[1], ready_fd, config); + target_security[1], ready_fd, diagnostic_fd, config); } namespace_owner owned_target(target_pid, target_pidfd); + stage = "namespace-target-parent-close"; + errno = 0; close_checked(target_mapping_pipe[0], "namespace target mapping reader"); close_checked(target_ready_pipe[1], "namespace target readiness writer"); close_checked(target_security[1], "namespace target security descriptor"); + stage = "namespace-target-bound-read"; char target_ready = 0; ssize_t target_ready_size; do { target_ready_size = read(target_ready_pipe[0], &target_ready, 1); } while (target_ready_size < 0 && errno == EINTR); if (target_ready_size != 1 || target_ready != 'B') { - _exit(125); + fail_stage(diagnostic_fd, stage, target_ready_size < 0 ? errno : 0); } + stage = "namespace-target-process-open"; + errno = 0; const std::string target_process_path = "/proc/" + std::to_string(target_pid); const int target_process_directory = open( target_process_path.c_str(), O_PATH | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); if (target_process_directory < 0 || pidfd_has_exited(target_pidfd)) { - _exit(125); + fail_stage( + diagnostic_fd, stage, + target_process_directory < 0 ? errno : 0); } + stage = "namespace-target-map-setgroups"; + errno = 0; write_mapping_file(target_process_directory, "setgroups", "deny\n"); + stage = "namespace-target-map-uid"; + errno = 0; write_mapping_file(target_process_directory, "uid_map", "65534 0 1\n"); + stage = "namespace-target-map-gid"; + errno = 0; write_mapping_file(target_process_directory, "gid_map", "65534 0 1\n"); + stage = "namespace-target-process-close"; + errno = 0; close_checked(target_process_directory, "target process directory"); + stage = "namespace-target-mapping-release"; + errno = 0; write_all(target_mapping_pipe[1], "M", 1); + stage = "namespace-target-mapping-close"; + errno = 0; close_checked(target_mapping_pipe[1], "namespace target mapping writer"); + stage = "namespace-target-filter-receive"; + errno = 0; const int target_listener = receive_descriptor(target_security[0], "FILTER"); + stage = "namespace-target-probe-verify"; + errno = 0; verify_target_isolation_probes(target_listener); + stage = "namespace-target-verified-read"; + errno = 0; if (receive_packet(target_security[0]) != "VERIFIED") { - _exit(125); + fail_stage(diagnostic_fd, stage, 0); } + stage = "namespace-parent-death-verify"; + errno = 0; require_parent_death(0); + stage = "namespace-identity-verify"; + errno = 0; if (prctl(PR_GET_DUMPABLE) != 0 || getsid(0) != getpid() || getpgrp() != getpid()) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } + stage = "namespace-target-go"; + errno = 0; send_packet(target_security[0], "GO"); + stage = "namespace-target-security-close"; + errno = 0; close_checked(target_security[0], "namespace target security supervisor descriptor"); + stage = "namespace-target-isolation-read"; do { target_ready_size = read(target_ready_pipe[0], &target_ready, 1); } while (target_ready_size < 0 && errno == EINTR); + stage = "namespace-target-ready-close"; + errno = 0; close_checked(target_ready_pipe[0], "namespace target readiness reader"); + stage = "namespace-parent-death-reverify"; + errno = 0; require_parent_death(0); + stage = "namespace-target-isolation-verify"; + errno = 0; if (prctl(PR_GET_DUMPABLE) != 0 || getsid(0) != getpid() || getpgrp() != getpid() || target_ready_size != 1 || target_ready != 'I') { - _exit(125); + fail_stage(diagnostic_fd, stage, 0); } + stage = "namespace-isolation-ready"; + errno = 0; write_all(ready_fd, "I", 1); + stage = "namespace-target-wait"; + errno = 0; const int target_status = wait_for_isolated_target(owned_target, target_listener); + stage = "namespace-listener-close"; + errno = 0; close_checked(target_listener, "namespace target seccomp listener"); + stage = "namespace-target-pidfd-close"; + errno = 0; owned_target.close_pidfd(); + stage = "namespace-descendant-kill"; + errno = 0; if (kill(-1, SIGKILL) != 0 && errno != ESRCH) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } + stage = "namespace-descendant-reap"; + errno = 0; while (true) { const pid_t reaped = waitpid(-1, nullptr, 0); if (reaped > 0 || (reaped < 0 && errno == EINTR)) { @@ -966,13 +1238,17 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { if (reaped < 0 && errno == ECHILD) { break; } - _exit(125); + fail_stage(diagnostic_fd, stage, reaped < 0 ? errno : 0); } + stage = "namespace-complete"; + errno = 0; write_all(ready_fd, "C", 1); + stage = "namespace-ready-close"; + errno = 0; close_checked(ready_fd, "namespace readiness descriptor"); _exit(wait_status_exit_code(target_status)); } catch (...) { - _exit(125); + fail_stage(diagnostic_fd, stage, errno); } } @@ -1004,6 +1280,16 @@ int run_linux_helper(options config) { close(ready_pipe[1]); throw std::runtime_error("cannot create namespace mapping pipe"); } + int diagnostic_pipe[2] {-1, -1}; + if (pipe2(diagnostic_pipe, O_CLOEXEC | O_NONBLOCK) != 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + close(mapping_pipe[0]); + close(mapping_pipe[1]); + throw std::runtime_error("cannot create namespace diagnostic pipe"); + } const int helper_pidfd = static_cast(syscall(SYS_pidfd_open, getpid(), 0)); if (helper_pidfd < 0) { close(release_pipe[0]); @@ -1012,6 +1298,8 @@ int run_linux_helper(options config) { close(ready_pipe[1]); close(mapping_pipe[0]); close(mapping_pipe[1]); + close(diagnostic_pipe[0]); + close(diagnostic_pipe[1]); throw std::runtime_error("cannot open stable helper identity"); } int namespace_pidfd = -1; @@ -1028,28 +1316,54 @@ int run_linux_helper(options config) { close(ready_pipe[1]); close(mapping_pipe[0]); close(mapping_pipe[1]); + close(diagnostic_pipe[0]); + close(diagnostic_pipe[1]); close(helper_pidfd); throw std::runtime_error(std::string("cannot create target PID namespace: ") + std::strerror(errno)); } if (namespace_init == 0) { - close_checked(release_pipe[1], "namespace release writer"); - close_checked(ready_pipe[0], "namespace readiness reader"); - close_checked(mapping_pipe[1], "namespace mapping writer"); + if (close(release_pipe[1]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-release-writer-close", + errno); + } + if (close(ready_pipe[0]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-ready-reader-close", + errno); + } + if (close(mapping_pipe[1]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-mapping-writer-close", + errno); + } + if (close(diagnostic_pipe[0]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-diagnostic-reader-close", + errno); + } run_namespace_init( - release_pipe[0], ready_pipe[1], mapping_pipe[0], + release_pipe[0], ready_pipe[1], mapping_pipe[0], diagnostic_pipe[1], config.protocol_fd, helper_pidfd, config); } namespace_owner owned_namespace(namespace_init, namespace_pidfd); close_checked(release_pipe[0], "helper release reader"); close_checked(ready_pipe[1], "helper readiness writer"); close_checked(mapping_pipe[0], "helper mapping reader"); + close_checked(diagnostic_pipe[1], "helper diagnostic writer"); char ready = 0; ssize_t ready_size; do { ready_size = read(ready_pipe[0], &ready, 1); } while (ready_size < 0 && errno == EINTR); if (ready_size != 1 || ready != 'B') { - throw std::runtime_error("target PID namespace did not bind helper lifetime"); + throw_setup_failure( + diagnostic_pipe[0], + "target PID namespace did not bind helper lifetime"); } const std::string process_path = "/proc/" + std::to_string(namespace_init); const int process_directory = open( @@ -1075,7 +1389,9 @@ int run_linux_helper(options config) { } while (ready_size < 0 && errno == EINTR); close_checked(helper_pidfd, "helper self pidfd"); if (ready_size != 1 || ready != 'R') { - throw std::runtime_error("target PID namespace setup did not complete"); + throw_setup_failure( + diagnostic_pipe[0], + "target PID namespace setup did not complete"); } send_descriptor(config.protocol_fd, "PREPARED", owned_namespace.pidfd()); if (receive_packet(config.protocol_fd) != "EXEC") { @@ -1087,7 +1403,9 @@ int run_linux_helper(options config) { ready_size = read(ready_pipe[0], &ready, 1); } while (ready_size < 0 && errno == EINTR); if (ready_size != 1 || ready != 'I') { - throw std::runtime_error("target privilege isolation did not complete"); + throw_setup_failure( + diagnostic_pipe[0], + "target privilege isolation did not complete"); } send_packet(config.protocol_fd, "RELEASED"); const int status = owned_namespace.wait(); @@ -1096,8 +1414,11 @@ int run_linux_helper(options config) { } while (ready_size < 0 && errno == EINTR); close_checked(ready_pipe[0], "helper readiness reader"); if (ready_size != 1 || ready != 'C') { - throw std::runtime_error("target namespace teardown did not complete"); + throw_setup_failure( + diagnostic_pipe[0], + "target namespace teardown did not complete"); } + close_checked(diagnostic_pipe[0], "helper diagnostics reader"); owned_namespace.close_pidfd(); send_packet(config.protocol_fd, "COMPLETE"); return wait_status_exit_code(status); diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 5fe289216b5b..6652c8863ed9 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -143,6 +143,7 @@ class TraceError(RuntimeError): PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS = 5 +PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES = 65536 PROCESS_TREE_TERM_GRACE_SECONDS = 1 WINDOWS_CREATE_SUSPENDED = 0x00000004 WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 @@ -932,10 +933,11 @@ def __init__( def _receive_protocol(self, expected: bytes, *, receive_pidfd: bool = False) -> None: item_size = array.array("i").itemsize + requested_flags = getattr(socket, "MSG_CMSG_CLOEXEC", 0) data, ancillary, flags, _address = self._protocol_socket.recvmsg( 128, socket.CMSG_SPACE(item_size), - getattr(socket, "MSG_CMSG_CLOEXEC", 0), + requested_flags, ) received = [] for level, kind, content in ancillary: @@ -943,7 +945,8 @@ def _receive_protocol(self, expected: bytes, *, receive_pidfd: bool = False) -> descriptor_bytes = array.array("i") descriptor_bytes.frombytes(content[:item_size]) received.extend(descriptor_bytes) - if flags != 0 or data != expected: + allowed_flags = {0, requested_flags} + if flags not in allowed_flags or data != expected: for descriptor in received: os.close(descriptor) raise TraceError( @@ -1017,6 +1020,35 @@ def close_streams(self) -> list[_IntegrityFailure]: setattr(self, attribute, None) return failures + def collect_startup_stderr(self) -> tuple[bytes, list[_IntegrityFailure]]: + if self._stderr_fd is None: + return b"", [] + descriptor = self._stderr_fd + self._stderr_fd = None + data = bytearray() + failures = [] + try: + while len(data) <= PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES: + chunk = os.read( + descriptor, + PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES + 1 - len(data), + ) + if not chunk: + break + data.extend(chunk) + if len(data) > PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES: + failures.append(_IntegrityFailure( + "linux-helper-stderr-bounds", + TraceError("Linux helper startup stderr exceeded its bound"), + )) + except BaseException as error: + failures.append(_IntegrityFailure("linux-helper-stderr-read", error)) + try: + os.close(descriptor) + except BaseException as error: + failures.append(_IntegrityFailure("linux-process-fd-close:stderr", error)) + return bytes(data[:PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES]), failures + def poll(self) -> int | None: if self.returncode is not None: return self.returncode @@ -1400,6 +1432,7 @@ def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _P return containment except BaseException as primary_error: failures = [] + startup_stderr = b"" if isinstance(primary_error, ExecutionIntegrityError): failures.extend(primary_error.secondary_errors) primary_error = primary_error.primary_error or primary_error @@ -1416,6 +1449,9 @@ def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _P else: cleanup = process.abort_blocked() failures.extend(cleanup.failures) + if cleanup.quiescence_proven: + startup_stderr, stderr_failures = process.collect_startup_stderr() + failures.extend(stderr_failures) close_failures = _close_process_containment( failed_containment, quiescence_proven=cleanup.quiescence_proven, @@ -1426,7 +1462,9 @@ def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _P if failures: raise ExecutionIntegrityError( f"Linux containment startup primary failure " - f"[{type(primary_error).__name__}: {primary_error}]; " + f"[{type(primary_error).__name__}: {primary_error}]" + + (f"; helper stderr {startup_stderr!r}" if startup_stderr else "") + + "; " f"secondary integrity failures: {_format_integrity_failures(failures)}", primary_error=primary_error, secondary_errors=failures, @@ -1435,7 +1473,8 @@ def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _P if process is not None: raise ExecutionIntegrityError( f"Linux containment startup primary failure " - f"[{type(primary_error).__name__}: {primary_error}]", + f"[{type(primary_error).__name__}: {primary_error}]" + + (f"; helper stderr {startup_stderr!r}" if startup_stderr else ""), primary_error=primary_error, secondary_errors=[], quiescence_proven=False, From b322ddf9654c7f2d95795062f4ec2017af8882dc Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 19:39:48 -0700 Subject: [PATCH 45/56] trace : validate Linux protocol descriptors Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 285 ++++++++++++++++++++++- tools/deepseek-v41-trace/trace_format.py | 87 +++++-- 2 files changed, 353 insertions(+), 19 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 7d0f7c81543e..632baf4d863d 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2878,15 +2878,54 @@ def test_linux_protocol_accepts_echoed_cmsg_cloexec(self) -> None: process._receive_protocol(b"READY") protocol.recvmsg.assert_called_once_with( 128, - trace.socket.CMSG_SPACE(array.array("i").itemsize), + trace.socket.CMSG_SPACE( + array.array("i").itemsize * + trace.LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS), + 1073741824, + ) + + def test_linux_protocol_accepts_one_cloexec_pidfd(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace, "_linux_fd_is_close_on_exec", return_value=True) as cloexec: + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(process.namespace_pidfd, 71) + cloexec.assert_called_once_with(71) + protocol.recvmsg.assert_called_once_with( + 128, + trace.socket.CMSG_SPACE( + array.array("i").itemsize * + trace.LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS), 1073741824, ) def test_linux_protocol_rejects_truncation_and_unknown_flags(self) -> None: + rights = array.array("i", [71, 72]).tobytes() for flags in (1073741824 | 32, 1073741824 | 8, 536870912): with self.subTest(flags=flags): protocol = mock.Mock() - protocol.recvmsg.return_value = (b"READY", [], flags, None) + protocol.recvmsg.return_value = ( + b"READY", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + flags, + None, + ) process = trace._LinuxNativeHelperProcess( ["approved"], 71, @@ -2896,9 +2935,247 @@ def test_linux_protocol_rejects_truncation_and_unknown_flags(self) -> None: stderr_fd=None, ) with mock.patch.object( - trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), self.assertRaisesRegex( - trace.TraceError, "protocol expected READY"): + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "unexpected flags"): process._receive_protocol(b"READY") + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_rejects_two_pidfds_in_one_record(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, mock.patch.object( + trace, "_linux_fd_is_close_on_exec") as cloexec, self.assertRaisesRegex( + trace.TraceError, "one namespace pidfd"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + cloexec.assert_not_called() + + def test_linux_protocol_rejects_multiple_rights_records(self) -> None: + first = array.array("i", [71]).tobytes() + second = array.array("i", [72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [ + (trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, first), + (trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, second), + ], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "multiple descriptor records"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_rejects_malformed_rights_payload(self) -> None: + malformed = array.array("i", [71]).tobytes() + b"x" + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, malformed)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "malformed descriptor data"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + close.assert_called_once_with(71) + + def test_linux_protocol_rejects_unexpected_ancillary_and_closes_rights(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [ + (trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights), + (trace.socket.SOL_SOCKET, 12345, b"unexpected"), + ], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "unexpected ancillary data"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + close.assert_called_once_with(71) + + def test_linux_protocol_rejects_pidfd_without_cloexec(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace, "_linux_fd_is_close_on_exec", return_value=False), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "not close-on-exec"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + close.assert_called_once_with(71) + + def test_linux_protocol_fcntl_failure_closes_pidfd(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + primary = OSError("fcntl failed") + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace, "_linux_fd_is_close_on_exec", side_effect=primary), mock.patch.object( + trace.os, "close") as close, self.assertRaises( + trace.ExecutionIntegrityError) as raised: + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertIs(raised.exception.primary_error, primary) + self.assertFalse(raised.exception.quiescence_proven) + close.assert_called_once_with(71) + + def test_linux_protocol_wrong_payload_closes_every_received_fd(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"WRONG", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "protocol expected PREPARED"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_close_failure_still_closes_remaining_fds(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"WRONG", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + close_error = OSError("close failed") + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close", side_effect=[close_error, None]) as close, self.assertRaises( + trace.ExecutionIntegrityError) as raised: + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-helper-received-fd-close"], + ) + self.assertFalse(raised.exception.quiescence_proven) + + def test_linux_protocol_rejects_descriptor_on_payload_only_message(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"READY", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "unexpected descriptor"): + process._receive_protocol(b"READY") + close.assert_called_once_with(71) def test_native_pidfd_packet_rejects_truncation_unknown_flags_and_oversize(self) -> None: import socket diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 6652c8863ed9..23ae024047f6 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -144,6 +144,7 @@ class TraceError(RuntimeError): PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS = 5 PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES = 65536 +LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS = 8 PROCESS_TREE_TERM_GRACE_SECONDS = 1 WINDOWS_CREATE_SUSPENDED = 0x00000004 WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 @@ -908,6 +909,12 @@ def _test_only_process_group_containment() -> Iterable[None]: _TEST_PROCESS_GROUP_CONTAINMENT.enabled = previous +def _linux_fd_is_close_on_exec(descriptor: int) -> bool: + import fcntl + + return bool(fcntl.fcntl(descriptor, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + + class _LinuxNativeHelperProcess: def __init__( self, @@ -936,31 +943,81 @@ def _receive_protocol(self, expected: bytes, *, receive_pidfd: bool = False) -> requested_flags = getattr(socket, "MSG_CMSG_CLOEXEC", 0) data, ancillary, flags, _address = self._protocol_socket.recvmsg( 128, - socket.CMSG_SPACE(item_size), + socket.CMSG_SPACE(item_size * LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS), requested_flags, ) received = [] + rights_records = 0 + ancillary_error = None for level, kind, content in ancillary: - if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: - descriptor_bytes = array.array("i") - descriptor_bytes.frombytes(content[:item_size]) - received.extend(descriptor_bytes) - allowed_flags = {0, requested_flags} - if flags not in allowed_flags or data != expected: + if level != socket.SOL_SOCKET or kind != socket.SCM_RIGHTS: + ancillary_error = "Linux containment helper sent unexpected ancillary data" + continue + rights_records += 1 + complete_size = len(content) - len(content) % item_size + descriptor_bytes = array.array("i") + descriptor_bytes.frombytes(content[:complete_size]) + received.extend(descriptor_bytes) + if len(content) == 0 or complete_size != len(content): + ancillary_error = "Linux containment helper sent malformed descriptor data" + + def reject(message: str) -> None: + failures = [] for descriptor in received: - os.close(descriptor) - raise TraceError( + try: + os.close(descriptor) + except BaseException as error: + failures.append(_IntegrityFailure( + "linux-helper-received-fd-close", error)) + primary = TraceError(message) + if failures: + raise ExecutionIntegrityError( + f"{message}; secondary integrity failures: " + f"{_format_integrity_failures(failures)}", + primary_error=primary, + secondary_errors=failures, + quiescence_proven=False, + ) from primary + raise primary + + allowed_flags = {0, requested_flags} + if flags not in allowed_flags: + reject( + f"Linux containment helper protocol returned unexpected flags {flags}") + if data != expected: + reject( f"Linux containment helper protocol expected {expected.decode('ascii')}") + if ancillary_error is not None: + reject(ancillary_error) + if rights_records > 1: + reject("Linux containment helper sent multiple descriptor records") if receive_pidfd: if len(received) != 1: - for descriptor in received: + reject("Linux containment helper did not provide one namespace pidfd") + descriptor = received[0] + try: + close_on_exec = _linux_fd_is_close_on_exec(descriptor) + except BaseException as error: + failures = [] + try: os.close(descriptor) - raise TraceError("Linux containment helper did not provide one namespace pidfd") - self.namespace_pidfd = received[0] + except BaseException as close_error: + failures.append(_IntegrityFailure( + "linux-helper-received-fd-close", close_error)) + raise ExecutionIntegrityError( + f"Linux containment helper namespace pidfd validation failed " + f"[{type(error).__name__}: {error}]" + + (f"; secondary integrity failures: " + f"{_format_integrity_failures(failures)}" if failures else ""), + primary_error=error, + secondary_errors=failures, + quiescence_proven=False, + ) from error + if not close_on_exec: + reject("Linux containment helper namespace pidfd is not close-on-exec") + self.namespace_pidfd = descriptor elif received: - for descriptor in received: - os.close(descriptor) - raise TraceError("Linux containment helper sent an unexpected descriptor") + reject("Linux containment helper sent an unexpected descriptor") def release_exec(self) -> None: self._receive_protocol(b"READY") From 0c398a5e14e8d29e321614952f49cd212678016b Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Sun, 13 Sep 2026 21:12:25 -0700 Subject: [PATCH 46/56] trace : clear namespace groups before mapping Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 50 +++++++++++++++++++ .../linux-containment-helper.cpp | 26 ++++++++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 632baf4d863d..47e4e673f4dd 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -2547,6 +2547,53 @@ def test_linux_native_helper_boundary_precedes_target_release(self) -> None: helper_source.index('!= \"EXEC\"'), ) + def test_linux_native_helper_clears_groups_before_mapping_denial(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + namespace_start = helper_source.index("[[noreturn]] void run_namespace_init") + namespace_source = helper_source[ + namespace_start: + helper_source.index("int run_linux_helper") + ] + target_source = helper_source[ + helper_source.index("[[noreturn]] void run_target_bootstrap"): + helper_source.index("[[noreturn]] void run_namespace_init") + ] + self.assertEqual(namespace_source.count("setgroups(0, nullptr)"), 1) + self.assertLess( + namespace_source.index('stage = "namespace-groups-clear"'), + namespace_source.index('write_all(ready_fd, "B", 1)'), + ) + self.assertLess( + namespace_start + namespace_source.index('write_all(ready_fd, "B", 1)'), + helper_source.index('write_mapping_file(process_directory, "setgroups", "deny\\n")'), + ) + self.assertLess( + namespace_source.index('stage = "namespace-mapping-close"'), + namespace_source.index('stage = "namespace-groups-verify"'), + ) + self.assertLess( + namespace_source.index('stage = "namespace-groups-verify"'), + namespace_source.index('stage = "namespace-setresgid"'), + ) + self.assertNotIn( + "setgroups(0, nullptr)", + namespace_source[namespace_source.index('stage = "namespace-release-read"'):], + ) + self.assertLess( + target_source.index('stage = "target-mapping-close"'), + target_source.index('stage = "target-groups-verify"'), + ) + self.assertLess( + target_source.index('stage = "target-groups-verify"'), + target_source.index('stage = "target-privilege-drop"'), + ) + self.assertNotIn("setgroups(0, nullptr)", target_source) + self.assertIn("namespace_group_count < 0 ? errno : 0", namespace_source) + self.assertIn("target_group_count < 0 ? errno : 0", target_source) + def test_posix_spawn_does_not_run_registered_atfork_callback(self) -> None: with tempfile.TemporaryDirectory() as temp: marker = Path(temp) / "atfork-ran" @@ -3407,7 +3454,9 @@ def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> N required_stages = { "namespace-parent-death", "namespace-parent-identity", + "namespace-groups-clear", "namespace-mapping-read", + "namespace-groups-verify", "namespace-setresgid", "namespace-setresuid", "namespace-mount-private", @@ -3419,6 +3468,7 @@ def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> N "target-parent-death", "target-session", "target-mapping-read", + "target-groups-verify", "target-privilege-drop", "target-isolation-probes", "target-isolation-ready", diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp index d520906d21e4..b18bb6525aeb 100644 --- a/tools/deepseek-v41-trace/linux-containment-helper.cpp +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -950,6 +950,14 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "target-mapping-close"; errno = 0; close_checked(mapping_fd, "target mapping descriptor"); + stage = "target-groups-verify"; + errno = 0; + const int target_group_count = getgroups(0, nullptr); + if (target_group_count != 0) { + fail_stage( + diagnostic_fd, stage, + target_group_count < 0 ? errno : 0); + } stage = "target-privilege-drop"; errno = 0; const int listener = drop_target_privileges(); @@ -1019,6 +1027,11 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "namespace-protocol-close"; errno = 0; close_checked(protocol_fd, "namespace protocol descriptor"); + stage = "namespace-groups-clear"; + errno = 0; + if (setgroups(0, nullptr) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } stage = "namespace-bound"; errno = 0; write_all(ready_fd, "B", 1); @@ -1034,6 +1047,14 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "namespace-mapping-close"; errno = 0; close_checked(mapping_fd, "namespace mapping descriptor"); + stage = "namespace-groups-verify"; + errno = 0; + const int namespace_group_count = getgroups(0, nullptr); + if (namespace_group_count != 0) { + fail_stage( + diagnostic_fd, stage, + namespace_group_count < 0 ? errno : 0); + } stage = "namespace-setresgid"; errno = 0; if (setresgid(0, 0, 0) != 0) { @@ -1080,11 +1101,6 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "namespace-release-close"; errno = 0; close_checked(release_fd, "namespace release descriptor"); - stage = "namespace-setgroups"; - errno = 0; - if (setgroups(0, nullptr) != 0) { - fail_stage(diagnostic_fd, stage, errno); - } int target_mapping_pipe[2] {-1, -1}; int target_ready_pipe[2] {-1, -1}; int target_security[2] {-1, -1}; From e4beb4df29289918c4682f4d9cbd5fff0d30a1df Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 00:22:10 -0700 Subject: [PATCH 47/56] trace : require zero-group Linux service Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c0c969ac-67cf-44f1-ae4f-545f7d731360 --- tests/test-deepseek41-trace.py | 78 ++++++++++++++++++- tools/deepseek-v41-trace/README.md | 4 +- .../generate-containment-helper-receipt.py | 4 +- .../linux-containment-helper.cpp | 44 +++++++++-- tools/deepseek-v41-trace/trace_format.py | 28 ++++++- 5 files changed, 145 insertions(+), 13 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 632baf4d863d..6c695b85036a 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -591,10 +591,12 @@ def replace_watchdog_events(root: Path, phase: str, events: list[dict[str, objec def fixture_containment_helper(revision: str, digest: str = "d" * 64) -> dict[str, object]: return { "format": "dsv41-containment-helper", - "version": 1, + "version": 2, "revision": revision, "filename": "llama-deepseek-v41-containment-helper", "sha256": digest, + "launcher_policy": "zero-supplementary-groups-v1", + "supplementary_groups": [], } @@ -2547,6 +2549,67 @@ def test_linux_native_helper_boundary_precedes_target_release(self) -> None: helper_source.index('!= \"EXEC\"'), ) + def test_linux_native_helper_requires_zero_group_service(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + run_source = helper_source[ + helper_source.index("int run_linux_helper"): + helper_source.index("#endif\n\n}") + ] + init_source = helper_source[ + helper_source.index("[[noreturn]] void run_namespace_init"): + helper_source.index("int run_linux_helper") + ] + target_source = helper_source[ + helper_source.index("[[noreturn]] void run_target_bootstrap"): + helper_source.index("[[noreturn]] void run_namespace_init") + ] + self.assertIn("--check-launcher-groups", helper_source) + self.assertIn("supplementary-groups=0", helper_source) + self.assertIn( + 'require_zero_supplementary_groups("containment launcher");', + run_source, + ) + self.assertLess( + run_source.index('require_zero_supplementary_groups("containment launcher");'), + run_source.index("require_initial_signal_state();"), + ) + self.assertLess( + run_source.index("require_initial_signal_state();"), + run_source.index("CLONE_NEWUSER"), + ) + self.assertNotIn("setgroups(", helper_source) + self.assertLess( + init_source.index('stage = "namespace-groups-verify"'), + init_source.index('stage = "namespace-setresgid"'), + ) + self.assertLess( + target_source.index('stage = "target-groups-verify"'), + target_source.index('stage = "target-privilege-drop"'), + ) + parent_source = inspect.getsource(trace._start_linux_native_helper) + self.assertLess( + parent_source.index("_require_zero_supplementary_groups()"), + parent_source.index("_start_linux_native_helper_process"), + ) + + def test_linux_parent_requires_zero_group_service_before_spawn(self) -> None: + with mock.patch.object(trace.os, "getgroups", return_value=[]): + trace._require_zero_supplementary_groups() + with mock.patch.object(trace.os, "getgroups", return_value=[10, 39, 105]), ( + self.assertRaisesRegex( + trace.TraceError, + "requires zero supplementary groups; found 3")): + trace._require_zero_supplementary_groups() + query_error = PermissionError(1, "Operation not permitted") + with mock.patch.object(trace.os, "getgroups", side_effect=query_error), ( + self.assertRaisesRegex( + trace.TraceError, + "cannot query Linux containment launcher supplementary groups")): + trace._require_zero_supplementary_groups() + def test_posix_spawn_does_not_run_registered_atfork_callback(self) -> None: with tempfile.TemporaryDirectory() as temp: marker = Path(temp) / "atfork-ran" @@ -2677,6 +2740,7 @@ def test_linux_native_helper_without_pidfd_support_fails_before_spawn(self) -> N lock = mock.Mock() lock.acquire.return_value = True with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support", side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( @@ -2696,6 +2760,7 @@ def test_linux_helper_pidfd_failure_aborts_before_protocol_release(self) -> None quiescence_proven=False, ) with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", side_effect=launch_error), self.assertRaises( @@ -2716,6 +2781,7 @@ def test_linux_helper_pidfd_open_failure_reaps_blocked_helper(self) -> None: with mock.patch.object( trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( @@ -2752,6 +2818,7 @@ def test_linux_post_spawn_cleanup_preserves_all_failures_and_reaps(self) -> None with mock.patch.object( trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( @@ -2797,6 +2864,7 @@ def test_linux_failed_startup_reap_retains_locked_authority(self) -> None: with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( trace, "_LINUX_POISONED_CONTAINMENT", None), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( @@ -3233,6 +3301,7 @@ def test_linux_protocol_startup_failure_reaps_and_closes_streams(self) -> None: primary = trace.TraceError("protocol startup failed") cleanup = trace._ContainmentCleanup([], True) with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( @@ -3338,6 +3407,7 @@ def test_linux_namespace_authority_precedes_release_completion(self) -> None: ) process.release_exec.side_effect = lambda: events.append("protocol") with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", @@ -3408,6 +3478,7 @@ def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> N "namespace-parent-death", "namespace-parent-identity", "namespace-mapping-read", + "namespace-groups-verify", "namespace-setresgid", "namespace-setresuid", "namespace-mount-private", @@ -3419,6 +3490,7 @@ def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> N "target-parent-death", "target-session", "target-mapping-read", + "target-groups-verify", "target-privilege-drop", "target-isolation-probes", "target-isolation-ready", @@ -4029,7 +4101,9 @@ def test_containment_helper_policy_mutations_fail_closed(self) -> None: ("revision", "b" * 40), ("filename", "other-helper"), ("sha256", "not-a-digest"), - ("version", 2)): + ("version", 1), + ("launcher_policy", "unbound"), + ("supplementary_groups", [44])): with self.subTest(field=field): policy = fixture_prompt_builder_policy(b"prompt") policy["containment_helper"][field] = value diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 0006c8b90206..f0130007ca40 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,9 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, accepts only `EINVAL` when a locked inherited `/proc` cannot be detached, mounts a new `nosuid,nodev,noexec` procfs over `/proc`, and verifies `/proc/self` resolves to namespace PID 1. A separate bounded binary diagnostic pipe reports setup stage and errno without sharing protocol tokens or exposing host data. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, empties supplemental groups, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, and isolated session before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Linux production execution must use an administrator-provisioned dedicated service identity whose live supplementary-group list is empty. The service may select one trusted primary group for required device or file access, or use exact administrator-managed ACLs, but it must not add the execution identity to `render`, `video`, or other supplementary groups. A normal login session is not valid, and an unprivileged process cannot repair a nonempty list after launch. Python checks the live list before it opens helper descriptors, and the helper checks `getgroups(0, nullptr) == 0` before it sends `READY` or creates a namespace, then rechecks the inherited empty list after both user-namespace mapping boundaries and after its credential drop. The signed containment-helper receipt binds `zero-supplementary-groups-v1` and an exact empty group list. Administrators must run `llama-deepseek-v41-containment-helper --check-launcher-groups` as `ExecStartPre=` under the same `User=` and `Group=` as the matrix service; success prints `supplementary-groups=0`. `SupplementaryGroups=` can add groups and does not replace this runtime check. Any nonzero list or query failure exits 125 before target code. + +Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, accepts only `EINVAL` when a locked inherited `/proc` cannot be detached, mounts a new `nosuid,nodev,noexec` procfs over `/proc`, and verifies `/proc/self` resolves to namespace PID 1. A separate bounded binary diagnostic pipe reports setup stage and errno without sharing protocol tokens or exposing host data. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap verifies that the inherited supplementary-group list is still empty, locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, isolated session, and empty supplementary-group list before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If the dedicated zero-group service contract, unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/generate-containment-helper-receipt.py b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py index 073380a6d2d7..3731c7931dc6 100644 --- a/tools/deepseek-v41-trace/generate-containment-helper-receipt.py +++ b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py @@ -27,10 +27,12 @@ def main() -> int: helper = args.helper.resolve(strict=True) receipt = { "format": "dsv41-containment-helper", - "version": 1, + "version": 2, "revision": args.revision, "filename": helper.name, "sha256": sha256_file(helper), + "launcher_policy": "zero-supplementary-groups-v1", + "supplementary_groups": [], } content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp index d520906d21e4..8d4da109c13d 100644 --- a/tools/deepseek-v41-trace/linux-containment-helper.cpp +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -18,7 +18,6 @@ #if defined(__linux__) #include -#include #include #include #include @@ -128,6 +127,28 @@ constexpr uint32_t DIAGNOSTIC_MAGIC = 0x44535634U; constexpr uint16_t DIAGNOSTIC_VERSION = 1; constexpr size_t DIAGNOSTIC_STAGE_CAPACITY = 48; +int query_supplementary_group_count(int * error_number) noexcept { + errno = 0; + const int count = getgroups(0, nullptr); + *error_number = count < 0 ? errno : 0; + return count; +} + +void require_zero_supplementary_groups(const char * context) { + int error_number = 0; + const int count = query_supplementary_group_count(&error_number); + if (count < 0) { + throw std::runtime_error( + std::string("cannot query ") + context + " supplementary groups: " + + std::strerror(error_number)); + } + if (count != 0) { + throw std::runtime_error( + std::string(context) + " requires zero supplementary groups; found " + + std::to_string(count)); + } +} + struct failure_diagnostic { uint32_t magic; uint16_t version; @@ -950,6 +971,11 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "target-mapping-close"; errno = 0; close_checked(mapping_fd, "target mapping descriptor"); + stage = "target-groups-verify"; + int groups_error = 0; + if (query_supplementary_group_count(&groups_error) != 0) { + fail_stage(diagnostic_fd, stage, groups_error); + } stage = "target-privilege-drop"; errno = 0; const int listener = drop_target_privileges(); @@ -1034,6 +1060,11 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "namespace-mapping-close"; errno = 0; close_checked(mapping_fd, "namespace mapping descriptor"); + stage = "namespace-groups-verify"; + int groups_error = 0; + if (query_supplementary_group_count(&groups_error) != 0) { + fail_stage(diagnostic_fd, stage, groups_error); + } stage = "namespace-setresgid"; errno = 0; if (setresgid(0, 0, 0) != 0) { @@ -1080,11 +1111,6 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "namespace-release-close"; errno = 0; close_checked(release_fd, "namespace release descriptor"); - stage = "namespace-setgroups"; - errno = 0; - if (setgroups(0, nullptr) != 0) { - fail_stage(diagnostic_fd, stage, errno); - } int target_mapping_pipe[2] {-1, -1}; int target_ready_pipe[2] {-1, -1}; int target_security[2] {-1, -1}; @@ -1253,6 +1279,7 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { } int run_linux_helper(options config) { + require_zero_supplementary_groups("containment launcher"); require_initial_signal_state(); set_parent_death(config.expected_parent); close_unneeded_fds(config.keep_fds, config.protocol_fd); @@ -1435,6 +1462,11 @@ int main(int argc, char ** argv) { return 0; } #if defined(__linux__) + if (argc == 2 && std::strcmp(argv[1], "--check-launcher-groups") == 0) { + require_zero_supplementary_groups("containment launcher"); + std::cout << "supplementary-groups=0\n"; + return 0; + } return run_linux_helper(parse_options(argc, argv)); #else (void) argc; diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 23ae024047f6..669b8e2b5f6d 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -1238,6 +1238,17 @@ def _all_catchable_signals() -> set[int]: } +def _require_zero_supplementary_groups() -> None: + try: + groups = os.getgroups() + except OSError as error: + raise TraceError( + f"cannot query Linux containment launcher supplementary groups: {error}") from error + if groups: + raise TraceError( + f"Linux containment launcher requires zero supplementary groups; found {len(groups)}") + + def _start_linux_native_helper_process( command: list[str], launch: dict[str, Any], @@ -1454,6 +1465,7 @@ def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _P pidfd = None exec_released = False try: + _require_zero_supplementary_groups() _linux_require_pidfd_support() if len(_linux_task_ids()) != 1: raise TraceError("Linux native containment requires a single-threaded Python supervisor") @@ -2474,13 +2486,23 @@ def _validate_containment_helper_policy( raise TraceError(f"{label} containment helper receipt is missing") _require_exact_keys( record, - {"format", "version", "revision", "filename", "sha256"}, + { + "format", + "version", + "revision", + "filename", + "sha256", + "launcher_policy", + "supplementary_groups", + }, f"{label} containment helper receipt", ) - if record["format"] != "dsv41-containment-helper" or record["version"] != 1 or ( + if record["format"] != "dsv41-containment-helper" or record["version"] != 2 or ( record["revision"] != revision) or ( record["filename"] != "llama-deepseek-v41-containment-helper") or re.fullmatch( - r"[0-9a-f]{64}", record.get("sha256", "")) is None: + r"[0-9a-f]{64}", record.get("sha256", "")) is None or ( + record["launcher_policy"] != "zero-supplementary-groups-v1") or ( + record["supplementary_groups"] != []): raise TraceError(f"{label} containment helper receipt is invalid") result = dict(record) result["path"] = f"{install_root}/bin/{record['filename']}" From 3b5bdd5e9366ad8c218c9b4b49119067b859dfe3 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 00:26:24 -0700 Subject: [PATCH 48/56] trace : require zero-group Linux service Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 --- tests/test-deepseek41-trace.py | 78 ++++++++++++------- tools/deepseek-v41-trace/README.md | 4 +- .../generate-containment-helper-receipt.py | 4 +- .../linux-containment-helper.cpp | 52 ++++++++----- tools/deepseek-v41-trace/trace_format.py | 28 ++++++- 5 files changed, 116 insertions(+), 50 deletions(-) diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 47e4e673f4dd..6c695b85036a 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -591,10 +591,12 @@ def replace_watchdog_events(root: Path, phase: str, events: list[dict[str, objec def fixture_containment_helper(revision: str, digest: str = "d" * 64) -> dict[str, object]: return { "format": "dsv41-containment-helper", - "version": 1, + "version": 2, "revision": revision, "filename": "llama-deepseek-v41-containment-helper", "sha256": digest, + "launcher_policy": "zero-supplementary-groups-v1", + "supplementary_groups": [], } @@ -2547,52 +2549,66 @@ def test_linux_native_helper_boundary_precedes_target_release(self) -> None: helper_source.index('!= \"EXEC\"'), ) - def test_linux_native_helper_clears_groups_before_mapping_denial(self) -> None: + def test_linux_native_helper_requires_zero_group_service(self) -> None: helper_source = ( Path(__file__).parents[1] / "tools/deepseek-v41-trace/linux-containment-helper.cpp" ).read_text(encoding="ascii") - namespace_start = helper_source.index("[[noreturn]] void run_namespace_init") - namespace_source = helper_source[ - namespace_start: + run_source = helper_source[ + helper_source.index("int run_linux_helper"): + helper_source.index("#endif\n\n}") + ] + init_source = helper_source[ + helper_source.index("[[noreturn]] void run_namespace_init"): helper_source.index("int run_linux_helper") ] target_source = helper_source[ helper_source.index("[[noreturn]] void run_target_bootstrap"): helper_source.index("[[noreturn]] void run_namespace_init") ] - self.assertEqual(namespace_source.count("setgroups(0, nullptr)"), 1) - self.assertLess( - namespace_source.index('stage = "namespace-groups-clear"'), - namespace_source.index('write_all(ready_fd, "B", 1)'), + self.assertIn("--check-launcher-groups", helper_source) + self.assertIn("supplementary-groups=0", helper_source) + self.assertIn( + 'require_zero_supplementary_groups("containment launcher");', + run_source, ) self.assertLess( - namespace_start + namespace_source.index('write_all(ready_fd, "B", 1)'), - helper_source.index('write_mapping_file(process_directory, "setgroups", "deny\\n")'), + run_source.index('require_zero_supplementary_groups("containment launcher");'), + run_source.index("require_initial_signal_state();"), ) self.assertLess( - namespace_source.index('stage = "namespace-mapping-close"'), - namespace_source.index('stage = "namespace-groups-verify"'), + run_source.index("require_initial_signal_state();"), + run_source.index("CLONE_NEWUSER"), ) + self.assertNotIn("setgroups(", helper_source) self.assertLess( - namespace_source.index('stage = "namespace-groups-verify"'), - namespace_source.index('stage = "namespace-setresgid"'), - ) - self.assertNotIn( - "setgroups(0, nullptr)", - namespace_source[namespace_source.index('stage = "namespace-release-read"'):], + init_source.index('stage = "namespace-groups-verify"'), + init_source.index('stage = "namespace-setresgid"'), ) self.assertLess( - target_source.index('stage = "target-mapping-close"'), target_source.index('stage = "target-groups-verify"'), + target_source.index('stage = "target-privilege-drop"'), ) + parent_source = inspect.getsource(trace._start_linux_native_helper) self.assertLess( - target_source.index('stage = "target-groups-verify"'), - target_source.index('stage = "target-privilege-drop"'), + parent_source.index("_require_zero_supplementary_groups()"), + parent_source.index("_start_linux_native_helper_process"), ) - self.assertNotIn("setgroups(0, nullptr)", target_source) - self.assertIn("namespace_group_count < 0 ? errno : 0", namespace_source) - self.assertIn("target_group_count < 0 ? errno : 0", target_source) + + def test_linux_parent_requires_zero_group_service_before_spawn(self) -> None: + with mock.patch.object(trace.os, "getgroups", return_value=[]): + trace._require_zero_supplementary_groups() + with mock.patch.object(trace.os, "getgroups", return_value=[10, 39, 105]), ( + self.assertRaisesRegex( + trace.TraceError, + "requires zero supplementary groups; found 3")): + trace._require_zero_supplementary_groups() + query_error = PermissionError(1, "Operation not permitted") + with mock.patch.object(trace.os, "getgroups", side_effect=query_error), ( + self.assertRaisesRegex( + trace.TraceError, + "cannot query Linux containment launcher supplementary groups")): + trace._require_zero_supplementary_groups() def test_posix_spawn_does_not_run_registered_atfork_callback(self) -> None: with tempfile.TemporaryDirectory() as temp: @@ -2724,6 +2740,7 @@ def test_linux_native_helper_without_pidfd_support_fails_before_spawn(self) -> N lock = mock.Mock() lock.acquire.return_value = True with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support", side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( @@ -2743,6 +2760,7 @@ def test_linux_helper_pidfd_failure_aborts_before_protocol_release(self) -> None quiescence_proven=False, ) with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", side_effect=launch_error), self.assertRaises( @@ -2763,6 +2781,7 @@ def test_linux_helper_pidfd_open_failure_reaps_blocked_helper(self) -> None: with mock.patch.object( trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( @@ -2799,6 +2818,7 @@ def test_linux_post_spawn_cleanup_preserves_all_failures_and_reaps(self) -> None with mock.patch.object( trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( @@ -2844,6 +2864,7 @@ def test_linux_failed_startup_reap_retains_locked_authority(self) -> None: with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( trace, "_LINUX_POISONED_CONTAINMENT", None), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( @@ -3280,6 +3301,7 @@ def test_linux_protocol_startup_failure_reaps_and_closes_streams(self) -> None: primary = trace.TraceError("protocol startup failed") cleanup = trace._ContainmentCleanup([], True) with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( @@ -3385,6 +3407,7 @@ def test_linux_namespace_authority_precedes_release_completion(self) -> None: ) process.release_exec.side_effect = lambda: events.append("protocol") with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( trace, "_linux_require_pidfd_support"), mock.patch.object( trace, "_linux_task_ids", return_value={1}), mock.patch.object( trace, "_start_linux_native_helper_process", @@ -3454,7 +3477,6 @@ def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> N required_stages = { "namespace-parent-death", "namespace-parent-identity", - "namespace-groups-clear", "namespace-mapping-read", "namespace-groups-verify", "namespace-setresgid", @@ -4079,7 +4101,9 @@ def test_containment_helper_policy_mutations_fail_closed(self) -> None: ("revision", "b" * 40), ("filename", "other-helper"), ("sha256", "not-a-digest"), - ("version", 2)): + ("version", 1), + ("launcher_policy", "unbound"), + ("supplementary_groups", [44])): with self.subTest(field=field): policy = fixture_prompt_builder_policy(b"prompt") policy["containment_helper"][field] = value diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index 0006c8b90206..f0130007ca40 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -172,7 +172,9 @@ Production installs no manifest-writing or runtime-path probe option. With `LLAM The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. -Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, accepts only `EINVAL` when a locked inherited `/proc` cannot be detached, mounts a new `nosuid,nodev,noexec` procfs over `/proc`, and verifies `/proc/self` resolves to namespace PID 1. A separate bounded binary diagnostic pipe reports setup stage and errno without sharing protocol tokens or exposing host data. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, empties supplemental groups, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, and isolated session before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. +Linux production execution must use an administrator-provisioned dedicated service identity whose live supplementary-group list is empty. The service may select one trusted primary group for required device or file access, or use exact administrator-managed ACLs, but it must not add the execution identity to `render`, `video`, or other supplementary groups. A normal login session is not valid, and an unprivileged process cannot repair a nonempty list after launch. Python checks the live list before it opens helper descriptors, and the helper checks `getgroups(0, nullptr) == 0` before it sends `READY` or creates a namespace, then rechecks the inherited empty list after both user-namespace mapping boundaries and after its credential drop. The signed containment-helper receipt binds `zero-supplementary-groups-v1` and an exact empty group list. Administrators must run `llama-deepseek-v41-containment-helper --check-launcher-groups` as `ExecStartPre=` under the same `User=` and `Group=` as the matrix service; success prints `supplementary-groups=0`. `SupplementaryGroups=` can add groups and does not replace this runtime check. Any nonzero list or query failure exits 125 before target code. + +Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, accepts only `EINVAL` when a locked inherited `/proc` cannot be detached, mounts a new `nosuid,nodev,noexec` procfs over `/proc`, and verifies `/proc/self` resolves to namespace PID 1. A separate bounded binary diagnostic pipe reports setup stage and errno without sharing protocol tokens or exposing host data. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap verifies that the inherited supplementary-group list is still empty, locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, isolated session, and empty supplementary-group list before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If the dedicated zero-group service contract, unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. diff --git a/tools/deepseek-v41-trace/generate-containment-helper-receipt.py b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py index 073380a6d2d7..3731c7931dc6 100644 --- a/tools/deepseek-v41-trace/generate-containment-helper-receipt.py +++ b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py @@ -27,10 +27,12 @@ def main() -> int: helper = args.helper.resolve(strict=True) receipt = { "format": "dsv41-containment-helper", - "version": 1, + "version": 2, "revision": args.revision, "filename": helper.name, "sha256": sha256_file(helper), + "launcher_policy": "zero-supplementary-groups-v1", + "supplementary_groups": [], } content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp index b18bb6525aeb..8d4da109c13d 100644 --- a/tools/deepseek-v41-trace/linux-containment-helper.cpp +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -18,7 +18,6 @@ #if defined(__linux__) #include -#include #include #include #include @@ -128,6 +127,28 @@ constexpr uint32_t DIAGNOSTIC_MAGIC = 0x44535634U; constexpr uint16_t DIAGNOSTIC_VERSION = 1; constexpr size_t DIAGNOSTIC_STAGE_CAPACITY = 48; +int query_supplementary_group_count(int * error_number) noexcept { + errno = 0; + const int count = getgroups(0, nullptr); + *error_number = count < 0 ? errno : 0; + return count; +} + +void require_zero_supplementary_groups(const char * context) { + int error_number = 0; + const int count = query_supplementary_group_count(&error_number); + if (count < 0) { + throw std::runtime_error( + std::string("cannot query ") + context + " supplementary groups: " + + std::strerror(error_number)); + } + if (count != 0) { + throw std::runtime_error( + std::string(context) + " requires zero supplementary groups; found " + + std::to_string(count)); + } +} + struct failure_diagnostic { uint32_t magic; uint16_t version; @@ -951,12 +972,9 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { errno = 0; close_checked(mapping_fd, "target mapping descriptor"); stage = "target-groups-verify"; - errno = 0; - const int target_group_count = getgroups(0, nullptr); - if (target_group_count != 0) { - fail_stage( - diagnostic_fd, stage, - target_group_count < 0 ? errno : 0); + int groups_error = 0; + if (query_supplementary_group_count(&groups_error) != 0) { + fail_stage(diagnostic_fd, stage, groups_error); } stage = "target-privilege-drop"; errno = 0; @@ -1027,11 +1045,6 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { stage = "namespace-protocol-close"; errno = 0; close_checked(protocol_fd, "namespace protocol descriptor"); - stage = "namespace-groups-clear"; - errno = 0; - if (setgroups(0, nullptr) != 0) { - fail_stage(diagnostic_fd, stage, errno); - } stage = "namespace-bound"; errno = 0; write_all(ready_fd, "B", 1); @@ -1048,12 +1061,9 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { errno = 0; close_checked(mapping_fd, "namespace mapping descriptor"); stage = "namespace-groups-verify"; - errno = 0; - const int namespace_group_count = getgroups(0, nullptr); - if (namespace_group_count != 0) { - fail_stage( - diagnostic_fd, stage, - namespace_group_count < 0 ? errno : 0); + int groups_error = 0; + if (query_supplementary_group_count(&groups_error) != 0) { + fail_stage(diagnostic_fd, stage, groups_error); } stage = "namespace-setresgid"; errno = 0; @@ -1269,6 +1279,7 @@ int wait_for_isolated_target(namespace_owner & target, int listener) { } int run_linux_helper(options config) { + require_zero_supplementary_groups("containment launcher"); require_initial_signal_state(); set_parent_death(config.expected_parent); close_unneeded_fds(config.keep_fds, config.protocol_fd); @@ -1451,6 +1462,11 @@ int main(int argc, char ** argv) { return 0; } #if defined(__linux__) + if (argc == 2 && std::strcmp(argv[1], "--check-launcher-groups") == 0) { + require_zero_supplementary_groups("containment launcher"); + std::cout << "supplementary-groups=0\n"; + return 0; + } return run_linux_helper(parse_options(argc, argv)); #else (void) argc; diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 23ae024047f6..669b8e2b5f6d 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -1238,6 +1238,17 @@ def _all_catchable_signals() -> set[int]: } +def _require_zero_supplementary_groups() -> None: + try: + groups = os.getgroups() + except OSError as error: + raise TraceError( + f"cannot query Linux containment launcher supplementary groups: {error}") from error + if groups: + raise TraceError( + f"Linux containment launcher requires zero supplementary groups; found {len(groups)}") + + def _start_linux_native_helper_process( command: list[str], launch: dict[str, Any], @@ -1454,6 +1465,7 @@ def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _P pidfd = None exec_released = False try: + _require_zero_supplementary_groups() _linux_require_pidfd_support() if len(_linux_task_ids()) != 1: raise TraceError("Linux native containment requires a single-threaded Python supervisor") @@ -2474,13 +2486,23 @@ def _validate_containment_helper_policy( raise TraceError(f"{label} containment helper receipt is missing") _require_exact_keys( record, - {"format", "version", "revision", "filename", "sha256"}, + { + "format", + "version", + "revision", + "filename", + "sha256", + "launcher_policy", + "supplementary_groups", + }, f"{label} containment helper receipt", ) - if record["format"] != "dsv41-containment-helper" or record["version"] != 1 or ( + if record["format"] != "dsv41-containment-helper" or record["version"] != 2 or ( record["revision"] != revision) or ( record["filename"] != "llama-deepseek-v41-containment-helper") or re.fullmatch( - r"[0-9a-f]{64}", record.get("sha256", "")) is None: + r"[0-9a-f]{64}", record.get("sha256", "")) is None or ( + record["launcher_policy"] != "zero-supplementary-groups-v1") or ( + record["supplementary_groups"] != []): raise TraceError(f"{label} containment helper receipt is invalid") result = dict(record) result["path"] = f"{install_root}/bin/{record['filename']}" From 927eab6e6c42ddda1df6fe01d34f46fa1fe34d4b Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 03:18:28 -0700 Subject: [PATCH 49/56] trace : bind replay session provenance Bind the canonical replay session to the accepted reconstruction without changing source bytes. Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> From 5f32d713cb276f3cbc100bf25ba903872a9f24d3 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 04:29:20 -0700 Subject: [PATCH 50/56] trace : bind model and watchdog identities Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/CMakeLists.txt | 1 + tests/test-deepseek41-trace.py | 375 ++++++++++++++++++++++- tools/deepseek-v41-trace/README.md | 4 + tools/deepseek-v41-trace/llama-trace.cpp | 289 ++++++++++++----- tools/deepseek-v41-trace/preflight.py | 134 +++++++- tools/deepseek-v41-trace/run_llama.py | 178 ++++++++++- tools/deepseek-v41-trace/run_matrix.py | 32 +- tools/deepseek-v41-trace/trace_format.py | 155 +++++++++- 8 files changed, 1068 insertions(+), 100 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 085334bcab61..340374ab2cd7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -225,6 +225,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) TEST test-deepseek41-trace APPEND PROPERTY ENVIRONMENT "DSV41_NATIVE_TRACE_BINARY=$" + "DSV41_NATIVE_CONTAINMENT_HELPER=$" "DSV41_NATIVE_MANIFEST_BINARY=$" "DSV41_NATIVE_INJECT_LIBRARY=$") endif() diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py index 6c695b85036a..375a140e22c6 100644 --- a/tests/test-deepseek41-trace.py +++ b/tests/test-deepseek41-trace.py @@ -3,6 +3,7 @@ import array import contextlib import copy +import hashlib import importlib.util import inspect import io @@ -498,6 +499,21 @@ def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, ob "audit_mode": 0o600, "audit_fd": 3, "audit_sha256": WATCHDOG_JSONL_SHA256, + "namespace_authority": { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": 9, + "host_procfs_root": "/proc", + "watchdog_pid": 123, + "watchdog_process_group_id": 122, + "watchdog_start_time_ticks": 456, + "watchdog_executable_path": "/usr/bin/python3", + "watchdog_command_sha256": "7" * 64, + "guardian_pid": 455, + "child_pid": 456, + "child_process_group_id": 455, + }, "audit": { "path": "", "sha256": WATCHDOG_JSONL_SHA256, @@ -825,12 +841,20 @@ def provenance_bytes( ) trust = fixture_install_trust(policy) runtime_build = fixture_runtime_build(policy) + source_root_lexical = Path(policy["source_root"]) + source_root_resolved = source_root_lexical.resolve(strict=False) + corpus_lexical = source_root_lexical / "tests" / "corpus" / "correctness-prose.txt" + corpus_resolved = source_root_resolved / "tests" / "corpus" / "correctness-prose.txt" record = { "format": "dsv41-prompt-provenance", - "version": 1, + "version": 2, "corpus_name": "correctness-prose.txt", "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], - "corpus_path": "/home/repo/tests/corpus/correctness-prose.txt", + "corpus_path": str(corpus_resolved), + "corpus_lexical_path": str(corpus_lexical), + "corpus_resolved_path": str(corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), "model_sha256": trace.MODEL_SHA256, "prompt_sha256": trace.sha256_bytes(prompt), "prompt_byte_count": len(prompt), @@ -1047,6 +1071,38 @@ def manifest( "remove_leading_bos_before_detokenize": True, "require_round_trip": True, }, + "model_file_identity": { + "format": "dsv41-model-file-identity", + "version": 1, + "path": storage["model"]["resolved_path"], + "device": 1, + "inode": 2, + "owner_uid": 1000, + "owner_gid": 1000, + "mode": 0o444, + "link_count": 1, + "byte_count": 123, + "modified_ns": 1, + "changed_ns": 1, + "status_flags": 0, + "source_descriptor_flags": 1, + "target_descriptor_flags": 0, + "sha256": trace.MODEL_SHA256, + }, + "watchdog_namespace": { + "format": "dsv41-watchdog-namespace-binding", + "version": 1, + "authority": "inherited-pidfd", + "host_watchdog_pid": 123, + "host_watchdog_process_group_id": 122, + "host_watchdog_start_time_ticks": 456, + "local_pid": 2, + "local_parent_pid": 1, + "local_process_group_id": 2, + "local_session_id": 2, + "namespace_pids": [1234, 2], + "private_procfs": True, + }, }) result["candidate"] = { "repository": trace.REPOSITORY, @@ -6341,6 +6397,21 @@ def wait(timeout: float | None = None) -> int: audit_event_count=2, procfs_root=procfs, ) + watchdog_audit["namespace_authority"] = { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": 9, + "host_procfs_root": "/proc", + "watchdog_pid": watchdog_pid, + "watchdog_process_group_id": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "watchdog_executable_path": str(Path(sys.executable).resolve()), + "watchdog_command_sha256": watchdog_audit["watchdog_command_sha256"], + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": guardian_pid, + } created_unix = int(now.timestamp()) audit = { "created_unix": created_unix, @@ -7186,14 +7257,16 @@ def test_python_runner_rejects_loader_overrides(self) -> None: with self.assertRaisesRegex(trace.TraceError, variable): trace.reject_loader_overrides() + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "source alias test") def test_prompt_builder_result_becomes_strict_provenance(self) -> None: with tempfile.TemporaryDirectory() as temp: - root = Path(temp) + root = Path(temp).resolve() builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" model = root / "model.gguf" corpus = root / "corpus.txt" source_root = Path(__file__).parents[1] source_corpus = source_root / "tests" / "corpus" / "correctness-prose.txt" + source_alias = root / "source-alias" output = root / "prompt.txt" tmpdir = root / "tmp" builder.parent.mkdir(parents=True) @@ -7201,6 +7274,7 @@ def test_prompt_builder_result_becomes_strict_provenance(self) -> None: builder.chmod(0o755) model.write_bytes(b"model") shutil.copyfile(source_corpus, corpus) + source_alias.symlink_to(source_root, target_is_directory=True) tmpdir.mkdir() builder = builder.resolve() model = model.resolve() @@ -7211,9 +7285,11 @@ def test_prompt_builder_result_becomes_strict_provenance(self) -> None: b"prompt", builder_path=str(builder.resolve()), builder_sha256=trace.sha256_file(builder), - source_root=str(source_root.resolve()), + source_root=str(source_alias), ) materialize_policy_runtime(builder_policy) + self.assertEqual(run_matrix.approved_source_root(builder_policy), source_root.resolve()) + self.assertEqual(run_llama.approved_source_root(builder_policy), source_root.resolve()) _validated, builder_policy_sha256 = trace.prompt_builder_approval( TEST_PROMPT_BUILDER_POLICY_ID, policies={TEST_PROMPT_BUILDER_POLICY_ID: builder_policy}, @@ -7277,10 +7353,16 @@ def run_builder(command, **_kwargs): ) provenance_path = Path(result["provenance_path"]) provenance = trace.strict_json_loads(provenance_path.read_text(encoding="ascii")) + self.assertEqual(provenance["source_root_lexical_path"], str(source_alias)) + self.assertEqual(provenance["source_root_resolved_path"], str(source_root.resolve())) + self.assertEqual(provenance["corpus_lexical_path"], str(source_corpus)) + self.assertEqual(provenance["corpus_resolved_path"], str(source_corpus.resolve())) self.assertEqual( set(provenance), { "format", "version", "corpus_name", "corpus_sha256", "corpus_path", + "corpus_lexical_path", "corpus_resolved_path", + "source_root_lexical_path", "source_root_resolved_path", "model_sha256", "prompt_sha256", "prompt_byte_count", "context", "decode_steps", "builder_approval_id", "builder_approval_sha256", "builder_path", "builder_sha256", "builder_revision", @@ -7305,6 +7387,291 @@ def run_builder(command, **_kwargs): path_resolver=lambda path, _label: path.resolve(), ) + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "descriptor identity test") + def test_model_descriptor_binds_bytes_and_rejects_path_replacement(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + model = root / "model.gguf" + replacement = root / "replacement.gguf" + model.write_bytes(b"approved model bytes") + replacement.write_bytes(b"replacement bytes") + descriptor, identity = run_llama.open_model_descriptor(model) + try: + self.assertEqual( + run_llama.sha256_descriptor(descriptor), + hashlib.sha256(b"approved model bytes").hexdigest(), + ) + os.replace(replacement, model) + self.assertEqual( + run_llama.sha256_descriptor(descriptor), + hashlib.sha256(b"approved model bytes").hexdigest(), + ) + if sys.platform == "linux" and os.environ.get("DSV41_NATIVE_TRACE_BINARY"): + environment = dict(os.environ) + environment["DSV41_MODEL_DESCRIPTOR"] = str(descriptor) + environment["DSV41_MODEL_DESCRIPTOR_IDENTITY"] = trace.canonical_json(identity) + native = subprocess.run( + [ + str(Path(os.environ["DSV41_NATIVE_TRACE_BINARY"]).resolve(strict=True)), + "--dsv41-test-model-descriptor", + str(model), + ], + check=False, + capture_output=True, + text=True, + env=environment, + pass_fds=(descriptor,), + ) + self.assertNotEqual(native.returncode, 0) + self.assertIn("held model descriptor policy is invalid", native.stderr) + with self.assertRaisesRegex( + run_llama.PreflightError, "linked pathname|pathname identity changed"): + run_llama.verify_model_descriptor(descriptor, identity) + finally: + os.close(descriptor) + + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "descriptor identity test") + def test_model_descriptor_rejects_symlink_and_in_place_mutation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + model = root / "model.gguf" + alias = root / "alias.gguf" + model.write_bytes(b"approved model bytes") + alias.symlink_to(model) + with self.assertRaisesRegex( + run_llama.PreflightError, "symbolic-link aliases"): + run_llama.open_model_descriptor(alias) + descriptor, identity = run_llama.open_model_descriptor(model) + try: + model.write_bytes(b"mutated model bytes") + with self.assertRaisesRegex( + run_llama.PreflightError, "identity or bytes changed"): + run_llama.verify_model_descriptor(descriptor, identity) + finally: + os.close(descriptor) + + @unittest.skipUnless(sys.platform == "linux", "watchdog pidfd authority test") + def test_watchdog_namespace_authority_rejects_reuse_and_executable_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + procfs = root / "proc" + watchdog_pid = 123 + guardian_pid = 456 + child_pid = 457 + descriptor = os.open("/dev/null", os.O_RDONLY) + + def proc_stat(pid: int, parent: int, group: int, start: int) -> str: + fields = ["S", str(parent), str(group), *(["0"] * 16), str(start)] + return f"{pid} (test) " + " ".join(fields) + "\n" + + try: + for pid in (watchdog_pid, guardian_pid, child_pid): + (procfs / str(pid)).mkdir(parents=True) + (procfs / "self" / "fdinfo").mkdir(parents=True) + (procfs / str(watchdog_pid) / "stat").write_text( + proc_stat(watchdog_pid, 1, watchdog_pid, 1000), encoding="ascii") + (procfs / str(guardian_pid) / "stat").write_text( + proc_stat(guardian_pid, watchdog_pid, guardian_pid, 2000), encoding="ascii") + (procfs / str(child_pid) / "stat").write_text( + proc_stat(child_pid, guardian_pid, guardian_pid, 3000), encoding="ascii") + executable = root / "python3" + executable.write_bytes(b"python") + (procfs / str(watchdog_pid) / "exe").symlink_to(executable) + command = b"python3\0watchdog.py\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(command) + def fake_pidfd_open(_pid: int, _flags: int) -> int: + retained = os.dup(descriptor) + (procfs / "self" / "fdinfo" / str(retained)).write_text( + f"Pid:\t{watchdog_pid}\n", encoding="ascii") + return retained + + audit = copy.deepcopy(AUDIT_RECORDS["watchdog"]["data"]) + audit.update({ + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "watchdog_executable_path": str(executable), + "watchdog_command_sha256": preflight.sha256_bytes(command), + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": guardian_pid, + }) + retained, authority = preflight.open_watchdog_namespace_authority( + audit, + procfs_root=procfs, + pidfd_open=fake_pidfd_open, + ) + try: + self.assertEqual(authority["watchdog_pid"], watchdog_pid) + preflight.verify_watchdog_namespace_authority( + retained, authority, procfs_root=procfs) + finally: + os.close(retained) + + reused = copy.deepcopy(audit) + reused["watchdog_start_time_ticks"] = 999 + with self.assertRaisesRegex(preflight.PreflightError, "start time"): + preflight.open_watchdog_namespace_authority( + reused, + procfs_root=procfs, + pidfd_open=fake_pidfd_open, + ) + other_executable = root / "other-python" + other_executable.write_bytes(b"other") + mismatched = copy.deepcopy(audit) + mismatched["watchdog_executable_path"] = str(other_executable) + with self.assertRaisesRegex(preflight.PreflightError, "executable"): + preflight.open_watchdog_namespace_authority( + mismatched, + procfs_root=procfs, + pidfd_open=fake_pidfd_open, + ) + finally: + os.close(descriptor) + + @unittest.skipUnless( + sys.platform == "linux" and + os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER") and + os.environ.get("DSV41_NATIVE_TRACE_BINARY"), + "native Linux watchdog namespace validation was not executed on this host", + ) + def test_native_watchdog_validation_crosses_private_pid_namespace(self) -> None: + import fcntl + + helper = Path(os.environ["DSV41_NATIVE_CONTAINMENT_HELPER"]).resolve(strict=True) + binary = Path(os.environ["DSV41_NATIVE_TRACE_BINARY"]).resolve(strict=True) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + lease = root / "watchdog.lease" + heartbeat = root / "watchdog.heartbeat" + audit = root / "watchdog.jsonl" + data_path = root / "watchdog-data.json" + lease.write_text("{}\n", encoding="ascii") + audit_line = '{"event":"preflight","timestamp":"1970-01-01T00:00:01.000Z"}\n' + audit.write_text(audit_line, encoding="ascii") + audit.chmod(0o600) + audit_descriptor = os.open(audit, os.O_RDONLY) + pidfd = os.pidfd_open(os.getpid()) + helper_descriptor = os.open(helper, os.O_RDONLY) + try: + fcntl.flock(audit_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + audit_status = audit.stat() + heartbeat.write_text( + json.dumps({ + "format": "strix-memory-watchdog-heartbeat", + "version": 2, + "lease_id": "1" * 32, + "sequence": 1, + "state": "active", + "updated_at": "1970-01-01T00:00:01.000Z", + "updated_monotonic_ns": time.monotonic_ns(), + "watchdog_pid": os.getpid(), + "watchdog_start_time_ticks": 1000, + "child_pid": os.getpid() + 2, + "child_process_group_id": os.getpid() + 1, + "sample": { + "audit_record_sha256": trace.sha256_bytes( + audit_line.encode("ascii")), + }, + }, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + command = ["python3", "run_matrix.py"] + data = copy.deepcopy(AUDIT_RECORDS["watchdog"]["data"]) + data.update({ + "lease_id": "1" * 32, + "lease_path": str(lease), + "watchdog_pid": os.getpid(), + "watchdog_start_time_ticks": 1000, + "watchdog_command_sha256": "7" * 64, + "watchdog_executable_path": str(Path(sys.executable).resolve()), + "watchdog_script_path": str( + (Path(__file__).parents[1] / "scripts" / "strix_memory_watchdog.py").resolve()), + "guardian_pid": os.getpid() + 1, + "child_pid": os.getpid() + 2, + "child_process_group_id": os.getpid() + 1, + "command": command, + "child_command_sha256": trace.sha256_bytes( + json.dumps(command, ensure_ascii=True, separators=(",", ":")).encode("ascii")), + "heartbeat_path": str(heartbeat), + "max_heartbeat_age_seconds": 5.0, + "audit_live_path": str(audit), + "audit_device": audit_status.st_dev, + "audit_inode": audit_status.st_ino, + "audit_uid": audit_status.st_uid, + "audit_mode": 0o600, + "audit_fd": audit_descriptor, + "namespace_authority": { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": pidfd, + "host_procfs_root": "/proc", + "watchdog_pid": os.getpid(), + "watchdog_process_group_id": os.getpgrp(), + "watchdog_start_time_ticks": 1000, + "watchdog_executable_path": str(Path(sys.executable).resolve()), + "watchdog_command_sha256": "7" * 64, + "guardian_pid": os.getpid() + 1, + "child_pid": os.getpid() + 2, + "child_process_group_id": os.getpid() + 1, + }, + }) + data_path.write_text( + json.dumps(data, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + environment = dict(os.environ) + environment.update({ + "DSV41_WATCHDOG_PIDFD": str(pidfd), + "STRIX_MEMORY_WATCHDOG_LEASE_PATH": str(lease), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH": str(heartbeat), + "STRIX_MEMORY_WATCHDOG_AUDIT_PATH": str(audit), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS": "5.0", + }) + contained = trace._run_contained_process( + [str(binary), "--dsv41-test-watchdog", str(data_path)], + label="native watchdog namespace test", + timeout=20, + input_data=None, + launch={ + "executable": str(binary), + "pass_fds": (pidfd,), + "_containment_helper_path": str(helper), + "_containment_helper_descriptor": helper_descriptor, + "env": environment, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + }, + ) + try: + if contained.primary_error is not None: + raise contained.primary_error + self.assertEqual(contained.integrity_failures, []) + self.assertIsNotNone(contained.result) + assert contained.result is not None + self.assertEqual( + contained.result.returncode, + 0, + contained.result.stderr.decode("utf-8", "replace"), + ) + binding = trace.strict_json_loads(contained.result.stdout.decode("ascii")) + self.assertTrue(binding["private_procfs"]) + self.assertEqual(binding["local_pid"], 2) + self.assertEqual(binding["local_parent_pid"], 1) + self.assertEqual(binding["local_pid"], binding["local_process_group_id"]) + self.assertEqual(binding["namespace_pids"][-1], binding["local_pid"]) + finally: + failures = trace._close_process_containment( + contained.containment, + quiescence_proven=contained.quiescence_proven, + ) + self.assertEqual(failures, []) + finally: + os.close(helper_descriptor) + os.close(pidfd) + fcntl.flock(audit_descriptor, fcntl.LOCK_UN) + os.close(audit_descriptor) + def test_rejects_cross_runtime_and_unknown_audit_envelopes(self) -> None: for mutation, message in ( (lambda value: value["audits"].update({"unknown": {}}), "audit envelope"), diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md index f0130007ca40..bca7e6b56591 100644 --- a/tools/deepseek-v41-trace/README.md +++ b/tools/deepseek-v41-trace/README.md @@ -21,6 +21,10 @@ The candidate exporter approval binds the exact producer revision, base revision Every approved install tree is verified before launch. The install owner is externally selected and must differ from the unprivileged execution identity. Every ancestor is canonical, non-symlinked, trusted-owned, ACL-free, and not writable by the execution identity or group/other. Every approved executable, containment helper, and receipt library is a canonical, ACL-free, non-writable, one-link regular file with the exact owner and SHA-256. On Linux, the exporter, prompt builder, and containment helper are opened without following the final symlink and launched through retained `/proc/self/fd` descriptors. Receipt-library descriptors remain open through process completion, while the immutable canonical install hierarchy prevents pathname substitution during loader consumption. Both native tools report the actual canonical loaded project-library paths and hashes; the prompt provenance and candidate manifest require the loaded closure to equal the approved receipt before and after protected work. Ordinary user-owned install smoke verifies layout, RPATH, helper receipt, version, and bytes only; it does not establish production trusted-root authorization. +The llama runner opens the approved model once without following aliases, hashes that held read-only descriptor before launch, retains it through Linux containment, and directs the exporter to load through `/proc/self/fd`. The runner and exporter bind device, inode, owner, mode, link count, size, timestamps, descriptor flags, and SHA-256 before model initialization and verify the same descriptor plus the original pathname after tracing. The watchdog is validated against host `/proc` before containment and represented inside the private PID namespace by a retained live pidfd. The signed audit binds the host PID, process group, start time, executable, command, guardian, and child identities, while the candidate manifest separately requires the exporter to be namespace-local PID 2 with parent PID 1, its own process group and session, matching NSpid, and private procfs. Host numeric PIDs are never interpreted as namespace-local PIDs. + +Prompt provenance version 2 records the lexical and resolved source root and corpus paths. The prompt builder accepts path aliases only when both resolve to the exact approved repository corpus and binds the canonical source identity into the signed provenance. + The signed execution authorization records the complete approval-policy SHA-256, verifier revision, tokenizer-policy SHA-256, and both selected approval record IDs, policy hashes, and install-trust hashes. Candidate-derived attestations and prompt provenance are evidence only and must exactly equal those external records. Seal creation performs the same semantic manifest, provenance, event, and coverage validation as standalone verification before invoking the signer. Every manifest and memory audit declares that the expert cache and KV cache are memory-resident and that there are no external cache or state paths. Missing, substituted, or additional file-backed cache/state declarations fail closed. diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp index c939d2c5ade1..3a25fc2c5753 100644 --- a/tools/deepseek-v41-trace/llama-trace.cpp +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -49,9 +49,11 @@ extern "C" { #if defined(__linux__) #include #include +#include #include #include #include +#include #endif #endif @@ -639,47 +641,138 @@ static std::string required_environment(const char * name) { #if defined(__linux__) static constexpr uint64_t DSV41_GIB = UINT64_C(1024)*1024*1024; -static std::pair proc_identity(int64_t pid) { - const std::vector bytes = read_file("/proc/" + std::to_string(pid) + "/stat"); - const std::string stat(bytes.begin(), bytes.end()); - const size_t command_end = stat.rfind(')'); - if (command_end == std::string::npos) { - throw std::runtime_error("watchdog process stat is invalid"); +static int required_descriptor(const char * name) { + const std::string value = required_environment(name); + size_t consumed = 0; + const long parsed = std::stol(value, &consumed); + if (consumed != value.size() || parsed <= STDERR_FILENO || + parsed > std::numeric_limits::max()) { + throw std::runtime_error(std::string("invalid descriptor environment: ") + name); } - std::istringstream fields(stat.substr(command_end + 2)); - std::string value; - int64_t parent = 0; - for (int field = 3; field <= 22; ++field) { - if (!(fields >> value)) { - throw std::runtime_error("watchdog process stat is truncated"); - } - if (field == 4) { - parent = std::stoll(value); + return static_cast(parsed); +} + +static std::string sha256_descriptor(int descriptor) { + sha256_t state; + sha256_init(&state); + std::vector buffer(8 * 1024 * 1024); + off_t offset = 0; + while (true) { + ssize_t count; + do { + count = pread(descriptor, buffer.data(), buffer.size(), offset); + } while (count < 0 && errno == EINTR); + if (count < 0) { + throw std::runtime_error("failed while hashing held model descriptor"); + } + if (count == 0) { + break; } + sha256_update(&state, buffer.data(), static_cast(count)); + offset += count; + } + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_final(&state, digest); + return sha256_hex(digest); +} + +static json model_descriptor_identity(int descriptor, const fs::path & model_path, bool hash_bytes) { + struct stat status {}; + const int status_flags = fcntl(descriptor, F_GETFL); + const int descriptor_flags = fcntl(descriptor, F_GETFD); + if (fstat(descriptor, &status) != 0 || status_flags < 0 || descriptor_flags < 0) { + throw std::runtime_error("cannot inspect held model descriptor"); + } + if (!S_ISREG(status.st_mode) || status.st_nlink < 1 || + (status_flags & O_ACCMODE) != O_RDONLY || descriptor_flags != 0) { + throw std::runtime_error("held model descriptor policy is invalid"); + } + json result = { + {"format", "dsv41-model-file-identity"}, + {"version", 1}, + {"path", model_path.string()}, + {"device", static_cast(status.st_dev)}, + {"inode", static_cast(status.st_ino)}, + {"owner_uid", static_cast(status.st_uid)}, + {"owner_gid", static_cast(status.st_gid)}, + {"mode", static_cast(status.st_mode & 07777)}, + {"link_count", static_cast(status.st_nlink)}, + {"byte_count", static_cast(status.st_size)}, + {"modified_ns", + static_cast(status.st_mtim.tv_sec)*INT64_C(1000000000) + status.st_mtim.tv_nsec}, + {"changed_ns", + static_cast(status.st_ctim.tv_sec)*INT64_C(1000000000) + status.st_ctim.tv_nsec}, + {"status_flags", status_flags}, + {"source_descriptor_flags", FD_CLOEXEC}, + {"target_descriptor_flags", 0}, + }; + if (hash_bytes) { + result["sha256"] = sha256_descriptor(descriptor); } - return { parent, std::stoull(value) }; + return result; } -static bool process_is_descendant(int64_t pid, int64_t ancestor) { - std::vector seen; - while (pid > 1 && std::find(seen.begin(), seen.end(), pid) == seen.end()) { - if (pid == ancestor) { - return true; +static json validate_model_descriptor(const fs::path & model_path, bool hash_bytes) { + const int descriptor = required_descriptor("DSV41_MODEL_DESCRIPTOR"); + json expected; + try { + expected = json::parse(required_environment("DSV41_MODEL_DESCRIPTOR_IDENTITY")); + } catch (const json::exception & error) { + throw std::runtime_error(std::string("model descriptor identity JSON is invalid: ") + error.what()); + } + const json observed = model_descriptor_identity(descriptor, model_path, hash_bytes); + if (expected != observed) { + throw std::runtime_error("held model descriptor differs from the runner identity"); + } + return observed; +} + +static std::vector namespace_pid_chain() { + std::ifstream status("/proc/self/status"); + if (!status) { + throw std::runtime_error("cannot read namespace-local process status"); + } + std::string line; + while (std::getline(status, line)) { + if (line.rfind("NSpid:", 0) != 0) { + continue; + } + std::istringstream values(line.substr(6)); + std::vector result; + int64_t value = 0; + while (values >> value) { + result.push_back(value); } - seen.push_back(pid); - pid = proc_identity(pid).first; + if (result.empty() || result.back() != getpid()) { + throw std::runtime_error("namespace-local NSpid identity is invalid"); + } + return result; } - return false; + throw std::runtime_error("namespace-local NSpid identity is missing"); } -static void validate_watchdog(const json & data) { +static void require_live_pidfd(int descriptor) { + if (fcntl(descriptor, F_GETFD) != 0) { + throw std::runtime_error("watchdog pidfd descriptor flags are invalid"); + } + std::array target {}; + const std::string descriptor_path = "/proc/self/fd/" + std::to_string(descriptor); + const ssize_t size = readlink(descriptor_path.c_str(), target.data(), target.size() - 1); + if (size <= 0 || std::string(target.data(), static_cast(size)) != "anon_inode:[pidfd]") { + throw std::runtime_error("watchdog authority descriptor is not a pidfd"); + } + pollfd observed {descriptor, POLLIN, 0}; + const int result = poll(&observed, 1, 0); + if (result < 0 || result != 0 || observed.revents != 0) { + throw std::runtime_error("watchdog authority pidfd is not live"); + } +} + +static json validate_watchdog(const json & data) { const int64_t pid = data.value("watchdog_pid", INT64_C(0)); const int64_t guardian_pid = data.value("guardian_pid", INT64_C(0)); const int64_t child_pid = data.value("child_pid", INT64_C(0)); const int64_t child_pgid = data.value("child_process_group_id", INT64_C(0)); - if (pid <= 1 || !fs::exists("/proc/" + std::to_string(pid))) { - throw std::runtime_error("watchdog process is not running"); - } if (data.value("format", "") != "strix-memory-watchdog-lease" || data.value("version", 0) != 2) { throw std::runtime_error("watchdog lease format is invalid"); } @@ -691,29 +784,36 @@ static void validate_watchdog(const json & data) { data.value("procfs_root", "") != "/proc") { throw std::runtime_error("watchdog execution policy is invalid"); } - if (guardian_pid <= 1 || child_pid <= 1 || child_pgid <= 1 || getpgrp() != child_pgid || - !process_is_descendant(getpid(), child_pid)) { - throw std::runtime_error("trace exporter is outside the watchdog-monitored process group"); - } - if (proc_identity(guardian_pid).first != pid || - proc_identity(child_pid).first != guardian_pid || - getpgid(guardian_pid) != child_pgid || - getpgid(child_pid) != child_pgid || - child_pgid != guardian_pid) { - throw std::runtime_error("watchdog guardian or child process identity is invalid"); - } - if (proc_identity(pid).second != data.value("watchdog_start_time_ticks", UINT64_C(0))) { - throw std::runtime_error("watchdog process start time changed"); - } - const std::vector command = read_file("/proc/" + std::to_string(pid) + "/cmdline"); - if (sha256_data(command.data(), command.size()) != data.value("watchdog_command_sha256", "")) { - throw std::runtime_error("watchdog process command changed"); - } - const fs::path executable_path = data.value("watchdog_executable_path", ""); - if (executable_path.empty() || - fs::canonical("/proc/" + std::to_string(pid) + "/exe") != fs::canonical(executable_path)) { - throw std::runtime_error("watchdog executable identity changed"); - } + if (pid <= 1 || guardian_pid <= 1 || child_pid <= 1 || child_pgid <= 1) { + throw std::runtime_error("watchdog host identity is invalid"); + } + const json authority = data.value("namespace_authority", json::object()); + const int authority_descriptor = required_descriptor("DSV41_WATCHDOG_PIDFD"); + if (authority.value("format", "") != "dsv41-watchdog-namespace-authority" || + authority.value("version", 0) != 1 || + authority.value("mechanism", "") != "inherited-pidfd" || + authority.value("descriptor", -1) != authority_descriptor || + authority.value("host_procfs_root", "") != "/proc" || + authority.value("watchdog_pid", INT64_C(0)) != pid || + authority.value("watchdog_start_time_ticks", UINT64_C(0)) != + data.value("watchdog_start_time_ticks", UINT64_C(0)) || + authority.value("watchdog_executable_path", "") != + data.value("watchdog_executable_path", "") || + authority.value("watchdog_command_sha256", "") != + data.value("watchdog_command_sha256", "") || + authority.value("guardian_pid", INT64_C(0)) != guardian_pid || + authority.value("child_pid", INT64_C(0)) != child_pid || + authority.value("child_process_group_id", INT64_C(0)) != child_pgid) { + throw std::runtime_error("watchdog namespace authority does not match the host audit"); + } + require_live_pidfd(authority_descriptor); + if (getpid() != 2 || getppid() != 1 || getpgrp() != getpid() || getsid(0) != getpid() || + canonical_path("/proc/self", "namespace-local process") != + canonical_path("/proc/" + std::to_string(getpid()), "namespace-local PID") || + !fs::is_directory("/proc/1")) { + throw std::runtime_error("trace exporter namespace-local process identity is invalid"); + } + const std::vector namespace_pids = namespace_pid_chain(); const fs::path script_path = data.value("watchdog_script_path", ""); if (script_path.empty() || data.value("watchdog_revision", "") != "778db6f50eae04e6c232c69b9575bdbd0747962b" || @@ -721,24 +821,6 @@ static void validate_watchdog(const json & data) { sha256_file(script_path) != WATCHDOG_SCRIPT_SHA256) { throw std::runtime_error("watchdog script identity changed"); } - std::vector watchdog_arguments; - size_t argument_start = 0; - while (argument_start < command.size()) { - const auto * begin = reinterpret_cast(command.data() + argument_start); - const size_t argument_size = std::char_traits::length(begin); - watchdog_arguments.emplace_back(begin, argument_size); - argument_start += argument_size + 1; - } - fs::path command_script; - if (watchdog_arguments.size() >= 2) { - command_script = watchdog_arguments[1]; - if (!command_script.is_absolute()) { - command_script = fs::canonical("/proc/" + std::to_string(pid) + "/cwd") / command_script; - } - } - if (watchdog_arguments.size() < 2 || fs::canonical(command_script) != fs::canonical(script_path)) { - throw std::runtime_error("watchdog script is not in executable argv position"); - } const fs::path heartbeat_path = data.value("heartbeat_path", ""); const double max_age = data.value("max_heartbeat_age_seconds", 0.0); if (heartbeat_path.empty() || max_age <= 0 || max_age > 30) { @@ -810,19 +892,12 @@ static void validate_watchdog(const json & data) { throw std::runtime_error("watchdog audit path is invalid"); } struct stat audit_stat; - struct stat descriptor_stat; const int audit_fd = data.value("audit_fd", -1); - const fs::path descriptor_path = - "/proc/" + std::to_string(pid) + "/fd/" + std::to_string(audit_fd); if (audit_fd < 0 || lstat(audit_path.c_str(), &audit_stat) != 0 || - stat(descriptor_path.c_str(), &descriptor_stat) != 0 || !S_ISREG(audit_stat.st_mode) || static_cast(audit_stat.st_dev) != data.value("audit_device", UINT64_C(0)) || static_cast(audit_stat.st_ino) != data.value("audit_inode", UINT64_C(0)) || - static_cast(descriptor_stat.st_dev) != data.value("audit_device", UINT64_C(0)) || - static_cast(descriptor_stat.st_ino) != data.value("audit_inode", UINT64_C(0)) || audit_stat.st_uid != getuid() || - static_cast(audit_stat.st_uid) != data.value("audit_uid", UINT64_C(0)) || (audit_stat.st_mode & 0777) != 0600 || data.value("audit_mode", UINT64_C(0)) != 0600) { throw std::runtime_error("watchdog persistent audit identity changed"); @@ -858,6 +933,23 @@ static void validate_watchdog(const json & data) { if (!found_audit_record) { throw std::runtime_error("watchdog heartbeat audit record is missing"); } + require_live_pidfd(authority_descriptor); + return { + {"format", "dsv41-watchdog-namespace-binding"}, + {"version", 1}, + {"authority", "inherited-pidfd"}, + {"host_watchdog_pid", pid}, + {"host_watchdog_process_group_id", + authority.value("watchdog_process_group_id", INT64_C(0))}, + {"host_watchdog_start_time_ticks", + data.value("watchdog_start_time_ticks", UINT64_C(0))}, + {"local_pid", getpid()}, + {"local_parent_pid", getppid()}, + {"local_process_group_id", getpgrp()}, + {"local_session_id", getsid(0)}, + {"namespace_pids", namespace_pids}, + {"private_procfs", true}, + }; } #endif @@ -891,16 +983,16 @@ static json audit_reference(const char * environment_name, const char * expected if (std::string(expected_kind) == "swap" && audit["data"].value("enabled", true)) { throw std::runtime_error("swap audit reports enabled swap"); } -#if defined(__linux__) - if (std::string(expected_kind) == "watchdog") { - validate_watchdog(audit["data"]); - } -#endif json result = { {"path", fs::absolute(path).lexically_normal().string()}, {"sha256", sha256_data(bytes.data(), bytes.size())}, {"created_unix", created}, }; +#if defined(__linux__) + if (std::string(expected_kind) == "watchdog") { + result["namespace_binding"] = validate_watchdog(audit["data"]); + } +#endif if (std::string(expected_kind) == "watchdog") { result["data"] = audit["data"]; } else if (std::string(expected_kind) == "memory") { @@ -1432,6 +1524,18 @@ int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); try { reject_loader_overrides(); +#if defined(__linux__) + if (argc == 3 && std::string(argv[1]) == "--dsv41-test-watchdog") { + const std::vector bytes = read_file(argv[2]); + std::cout << validate_watchdog(json::parse(bytes.begin(), bytes.end())).dump() << '\n'; + return 0; + } + if (argc == 3 && std::string(argv[1]) == "--dsv41-test-model-descriptor") { + const fs::path model_path = canonical_path(argv[2], "test model"); + std::cout << validate_model_descriptor(model_path, true).dump() << '\n'; + return 0; + } +#endif if (argc == 2 && std::string(argv[1]) == "--version") { common_init(); const fs::path executable = current_executable_path(); @@ -1555,6 +1659,13 @@ int main(int argc, char ** argv) { const fs::path model_path = model_storage.resolved_path; const fs::path prompt_path = prompt_storage.resolved_path; const fs::path output_path = output_storage.resolved_path; +#if !defined(__linux__) + throw std::runtime_error("trace model descriptor binding requires Linux"); +#else + const int model_descriptor = required_descriptor("DSV41_MODEL_DESCRIPTOR"); + const json model_file_identity = validate_model_descriptor(model_path, true); + params.model.path = "/proc/self/fd/" + std::to_string(model_descriptor); +#endif llama_backend_init(); llama_numa_init(params.numa); common_init_result_ptr init = common_init_from_params(params); @@ -1626,6 +1737,10 @@ int main(int argc, char ** argv) { {"expert_cache_slots", params.expert_cache_slots}, {"expert_cache_bytes", static_cast(params.expert_cache_mib) << 20}, {"tokenizer", tokenizer_policy}, +#if defined(__linux__) + {"model_file_identity", model_file_identity}, + {"watchdog_namespace", watchdog_audit["namespace_binding"]}, +#endif {"deepseek41", { {"layer_count", 40}, {"vocab_size", n_vocab}, @@ -1647,8 +1762,13 @@ int main(int argc, char ** argv) { {"model", { {"path", model_path.string()}, {"architecture", "deepseek41"}, +#if defined(__linux__) + {"byte_count", model_file_identity["byte_count"]}, + {"sha256", model_file_identity["sha256"]}, +#else {"byte_count", fs::file_size(model_path)}, {"sha256", sha256_file(model_path)}, +#endif }}, {"prompt", { {"path", prompt_path.string()}, @@ -1725,7 +1845,12 @@ int main(int argc, char ** argv) { } #if defined(__linux__) - validate_watchdog(watchdog_audit["data"]); + if (validate_watchdog(watchdog_audit["data"]) != watchdog_audit["namespace_binding"]) { + throw std::runtime_error("watchdog namespace binding changed during trace execution"); + } + if (validate_model_descriptor(model_path, true) != model_file_identity) { + throw std::runtime_error("held model descriptor changed during trace execution"); + } #endif end_loader_monitor(executable_path); writer.bind_runtime_post(runtime_libraries_json( diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py index d939dd26f478..e729102f39bf 100644 --- a/tools/deepseek-v41-trace/preflight.py +++ b/tools/deepseek-v41-trace/preflight.py @@ -507,6 +507,16 @@ def proc_parent_pid(stat: str) -> int: return int(fields[1]) +def proc_process_group_id(stat: str) -> int: + command_end = stat.rfind(")") + if command_end < 0: + raise PreflightError("process stat is invalid") + fields = stat[command_end + 2:].split() + if len(fields) < 3: + raise PreflightError("process stat is truncated") + return int(fields[2]) + + def is_descendant(pid: int, ancestor_pid: int, procfs_root: Path) -> bool: seen = set() while pid > 1 and pid not in seen: @@ -520,6 +530,114 @@ def is_descendant(pid: int, ancestor_pid: int, procfs_root: Path) -> bool: return False +def open_watchdog_namespace_authority( + watchdog: dict[str, object], + *, + procfs_root: Path = Path("/proc"), + pidfd_open: Callable[[int, int], int] | None = None, +) -> tuple[int, dict[str, object]]: + if sys.platform != "linux": + raise PreflightError("watchdog namespace authority requires Linux pidfds") + opener = pidfd_open or getattr(os, "pidfd_open", None) + if opener is None: + raise PreflightError("watchdog namespace authority requires os.pidfd_open") + try: + watchdog_pid = int(watchdog["watchdog_pid"]) + watchdog_start = int(watchdog["watchdog_start_time_ticks"]) + guardian_pid = int(watchdog["guardian_pid"]) + child_pid = int(watchdog["child_pid"]) + child_pgid = int(watchdog["child_process_group_id"]) + executable_path = str(watchdog["watchdog_executable_path"]) + command_sha256 = str(watchdog["watchdog_command_sha256"]) + except (KeyError, TypeError, ValueError) as error: + raise PreflightError(f"watchdog namespace authority identity is invalid: {error}") from error + descriptor = -1 + try: + descriptor = int(opener(watchdog_pid, 0)) + watchdog_stat = (procfs_root / str(watchdog_pid) / "stat").read_text(encoding="ascii") + guardian_stat = (procfs_root / str(guardian_pid) / "stat").read_text(encoding="ascii") + child_stat = (procfs_root / str(child_pid) / "stat").read_text(encoding="ascii") + live_executable = str((procfs_root / str(watchdog_pid) / "exe").resolve(strict=True)) + live_command = (procfs_root / str(watchdog_pid) / "cmdline").read_bytes() + fdinfo = (procfs_root / "self" / "fdinfo" / str(descriptor)).read_text(encoding="ascii") + fdinfo_pid = next( + int(line.split(":", 1)[1].strip()) + for line in fdinfo.splitlines() + if line.startswith("Pid:") + ) + if proc_start_time_ticks(watchdog_stat) != watchdog_start: + raise PreflightError("watchdog namespace authority start time does not match") + if live_executable != str(Path(executable_path).resolve(strict=True)): + raise PreflightError("watchdog namespace authority executable does not match") + if sha256_bytes(live_command) != command_sha256: + raise PreflightError("watchdog namespace authority command does not match") + if fdinfo_pid != watchdog_pid: + raise PreflightError("watchdog namespace authority pidfd does not match") + watchdog_pgid = proc_process_group_id(watchdog_stat) + if proc_parent_pid(guardian_stat) != watchdog_pid or ( + proc_process_group_id(guardian_stat) != child_pgid) or ( + guardian_pid != child_pgid) or ( + proc_parent_pid(child_stat) != guardian_pid) or ( + proc_process_group_id(child_stat) != child_pgid): + raise PreflightError("watchdog namespace authority process tree does not match") + authority = { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": descriptor, + "host_procfs_root": str(procfs_root.resolve(strict=True)), + "watchdog_pid": watchdog_pid, + "watchdog_process_group_id": watchdog_pgid, + "watchdog_start_time_ticks": watchdog_start, + "watchdog_executable_path": live_executable, + "watchdog_command_sha256": command_sha256, + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": child_pgid, + } + return descriptor, authority + except (OSError, StopIteration, ValueError) as error: + if descriptor >= 0: + os.close(descriptor) + raise PreflightError(f"cannot establish watchdog namespace authority: {error}") from error + except BaseException: + if descriptor >= 0: + os.close(descriptor) + raise + + +def verify_watchdog_namespace_authority( + descriptor: int, + authority: dict[str, object], + *, + procfs_root: Path = Path("/proc"), +) -> None: + try: + if authority.get("format") != "dsv41-watchdog-namespace-authority" or ( + authority.get("version") != 1) or authority.get("mechanism") != "inherited-pidfd" or ( + authority.get("descriptor") != descriptor): + raise PreflightError("watchdog namespace authority receipt is invalid") + watchdog_pid = int(authority["watchdog_pid"]) + watchdog_start = int(authority["watchdog_start_time_ticks"]) + stat_text = (procfs_root / str(watchdog_pid) / "stat").read_text(encoding="ascii") + executable = str((procfs_root / str(watchdog_pid) / "exe").resolve(strict=True)) + command = (procfs_root / str(watchdog_pid) / "cmdline").read_bytes() + fdinfo = (procfs_root / "self" / "fdinfo" / str(descriptor)).read_text(encoding="ascii") + fdinfo_pid = next( + int(line.split(":", 1)[1].strip()) + for line in fdinfo.splitlines() + if line.startswith("Pid:") + ) + if fdinfo_pid != watchdog_pid or proc_start_time_ticks(stat_text) != watchdog_start or ( + executable != authority.get("watchdog_executable_path")) or ( + sha256_bytes(command) != authority.get("watchdog_command_sha256")): + raise PreflightError("watchdog namespace authority changed") + except PreflightError: + raise + except (KeyError, OSError, StopIteration, TypeError, ValueError) as error: + raise PreflightError(f"cannot verify watchdog namespace authority: {error}") from error + + def read_heartbeat( path: Path, max_age_seconds: float, @@ -1365,12 +1483,18 @@ def validate_prompt_provenance( raise PreflightError(f"prompt provenance is invalid: {error}") from error if not isinstance(record, dict): raise PreflightError("prompt provenance must be a JSON object") + source_root_lexical = Path(str(builder_policy["source_root"])) + source_root_resolved = resolved(source_root_lexical) + corpus_resolved = resolved(source_root_resolved / "tests" / "corpus" / corpus_name) expected = { "format": "dsv41-prompt-provenance", - "version": 1, + "version": 2, "corpus_name": corpus_name, "corpus_sha256": corpus_sha256, - "corpus_path": f"{builder_policy['source_root']}/tests/corpus/{corpus_name}", + "corpus_path": str(corpus_resolved), + "corpus_resolved_path": str(corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), "model_sha256": model_sha256, "prompt_sha256": sha256_bytes(prompt_bytes), "prompt_byte_count": prompt_size, @@ -1390,6 +1514,7 @@ def validate_prompt_provenance( if record.get(key) != value: raise PreflightError(f"prompt provenance {key} mismatch") required = set(expected) | { + "corpus_lexical_path", "builder_runtime_build", "builder_runtime_build_sha256", "builder_install_trust", @@ -1397,6 +1522,11 @@ def validate_prompt_provenance( } if set(record) != required: raise PreflightError("prompt provenance fields are invalid") + try: + if resolved(Path(str(record["corpus_lexical_path"]))) != corpus_resolved: + raise PreflightError("prompt provenance corpus lexical path resolves outside the approved source") + except OSError as error: + raise PreflightError(f"prompt provenance corpus lexical path is invalid: {error}") from error try: runtime_build = validate_runtime_build_evidence( record["builder_runtime_build"], builder_policy, label="prompt builder") diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py index 24da2a85d829..40696c93b1d0 100644 --- a/tools/deepseek-v41-trace/run_llama.py +++ b/tools/deepseek-v41-trace/run_llama.py @@ -7,6 +7,7 @@ import os import re import shlex +import stat import subprocess import sys from pathlib import Path @@ -15,11 +16,13 @@ PreflightError, bind_embedded_audits, bind_prompt_provenance, + open_watchdog_namespace_authority, resolved, run_strix_preflight, safe_trace_path, seal_audits, validate_prompt_provenance, + verify_watchdog_namespace_authority, verify_sealed_audits, write_audits, ) @@ -65,6 +68,13 @@ ) +def approved_source_root(prompt_policy: dict[str, object]) -> Path: + try: + return Path(str(prompt_policy["source_root"])).expanduser().resolve(strict=True) + except (KeyError, OSError) as error: + raise PreflightError(f"prompt builder approved source root is invalid: {error}") from error + + def git_output(repo: Path, *args: str) -> bytes: try: return subprocess.check_output(["git", "-C", str(repo), *args], stderr=subprocess.STDOUT) @@ -72,6 +82,149 @@ def git_output(repo: Path, *args: str) -> bytes: raise PreflightError(f"git {' '.join(args)} failed: {error}") from error +def sha256_descriptor(descriptor: int) -> str: + digest = hashlib.sha256() + offset = 0 + try: + while True: + chunk = os.pread(descriptor, 8 * 1024 * 1024, offset) + if not chunk: + return digest.hexdigest() + digest.update(chunk) + offset += len(chunk) + except AttributeError as error: + raise PreflightError("descriptor hashing requires os.pread") from error + except OSError as error: + raise PreflightError(f"cannot hash held model descriptor: {error}") from error + + +def model_descriptor_identity( + descriptor: int, + path: Path, + *, + hash_bytes: bool, +) -> dict[str, object]: + try: + import fcntl + + record = os.fstat(descriptor) + status_flags = fcntl.fcntl(descriptor, fcntl.F_GETFL) + descriptor_flags = fcntl.fcntl(descriptor, fcntl.F_GETFD) + except (ImportError, OSError) as error: + raise PreflightError(f"cannot inspect held model descriptor: {error}") from error + if not stat.S_ISREG(record.st_mode): + raise PreflightError("model descriptor is not a regular file") + if record.st_nlink < 1: + raise PreflightError("model descriptor has no linked pathname") + if (status_flags & os.O_ACCMODE) != os.O_RDONLY: + raise PreflightError("model descriptor is not read-only") + identity = { + "format": "dsv41-model-file-identity", + "version": 1, + "path": str(path), + "device": record.st_dev, + "inode": record.st_ino, + "owner_uid": record.st_uid, + "owner_gid": record.st_gid, + "mode": stat.S_IMODE(record.st_mode), + "link_count": record.st_nlink, + "byte_count": record.st_size, + "modified_ns": record.st_mtime_ns, + "changed_ns": record.st_ctime_ns, + "status_flags": status_flags, + "source_descriptor_flags": descriptor_flags, + "target_descriptor_flags": 0, + } + if hash_bytes: + identity["sha256"] = sha256_descriptor(descriptor) + return identity + + +def open_model_descriptor(path: Path) -> tuple[int, dict[str, object]]: + lexical_path = path.expanduser() + if not lexical_path.is_absolute(): + lexical_path = Path.cwd() / lexical_path + try: + canonical_path = lexical_path.resolve(strict=True) + except OSError as error: + raise PreflightError(f"cannot resolve model path: {error}") from error + if canonical_path != lexical_path: + raise PreflightError("model path must not contain lexical or symbolic-link aliases") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = -1 + try: + descriptor = os.open(canonical_path, flags) + before = model_descriptor_identity(descriptor, canonical_path, hash_bytes=False) + identity = model_descriptor_identity(descriptor, canonical_path, hash_bytes=True) + after = model_descriptor_identity(descriptor, canonical_path, hash_bytes=False) + if before != after: + raise PreflightError("model descriptor identity changed while hashing") + path_record = canonical_path.stat(follow_symlinks=False) + if ( + path_record.st_dev, + path_record.st_ino, + path_record.st_uid, + path_record.st_gid, + stat.S_IMODE(path_record.st_mode), + path_record.st_nlink, + path_record.st_size, + path_record.st_mtime_ns, + path_record.st_ctime_ns, + ) != ( + identity["device"], + identity["inode"], + identity["owner_uid"], + identity["owner_gid"], + identity["mode"], + identity["link_count"], + identity["byte_count"], + identity["modified_ns"], + identity["changed_ns"], + ): + raise PreflightError("model pathname does not identify the held descriptor") + return descriptor, identity + except BaseException: + if descriptor >= 0: + os.close(descriptor) + raise + + +def verify_model_descriptor( + descriptor: int, + identity: dict[str, object], +) -> None: + path = Path(str(identity["path"])) + observed = model_descriptor_identity(descriptor, path, hash_bytes=True) + if observed != identity: + raise PreflightError("model descriptor identity or bytes changed during execution") + try: + path_record = path.stat(follow_symlinks=False) + except OSError as error: + raise PreflightError(f"cannot revalidate model pathname: {error}") from error + if ( + path_record.st_dev, + path_record.st_ino, + path_record.st_uid, + path_record.st_gid, + stat.S_IMODE(path_record.st_mode), + path_record.st_nlink, + path_record.st_size, + path_record.st_mtime_ns, + path_record.st_ctime_ns, + ) != ( + identity["device"], + identity["inode"], + identity["owner_uid"], + identity["owner_gid"], + identity["mode"], + identity["link_count"], + identity["byte_count"], + identity["modified_ns"], + identity["changed_ns"], + ): + raise PreflightError("model pathname identity changed during execution") + + def candidate_attestation( args: argparse.Namespace, exporter: Path, @@ -509,6 +662,8 @@ def main() -> int: parser.add_argument("--preflight-only", action="store_true") args = parser.parse_args() + model_descriptor = -1 + watchdog_descriptor = -1 try: validate_runtime_config(args) reject_loader_overrides() @@ -559,9 +714,10 @@ def main() -> int: if git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() != ( approval_policy.verifier_revision): raise PreflightError("candidate verifier checkout differs from the external approval policy") - if str(repo) != prompt_policy["source_root"] or candidate_policy["revision"] != prompt_policy["revision"]: + if repo != approved_source_root(prompt_policy) or candidate_policy["revision"] != prompt_policy["revision"]: raise PreflightError("candidate repository or revision differs from prompt builder approval") - model_sha256 = sha256_file(resolved(args.model)) + model_descriptor, model_identity = open_model_descriptor(args.model) + model_sha256 = str(model_identity["sha256"]) if model_sha256 != MODEL_SHA256: raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") provenance = validate_prompt_provenance( @@ -663,6 +819,9 @@ def main() -> int: "expert_cache_mib": args.expert_cache_mib, "gpu_layers": args.gpu_layers, } + watchdog_descriptor, watchdog_authority = open_watchdog_namespace_authority( + preflight_audit["watchdog"]) + preflight_audit["watchdog"]["namespace_authority"] = watchdog_authority pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) pre_audit_digests = seal_audits(pre_audits) environment = os.environ.copy() @@ -670,6 +829,9 @@ def main() -> int: environment["DSV41_TRACE_SWAP_AUDIT"] = pre_audits["swap"] environment["DSV41_TRACE_WATCHDOG_AUDIT"] = pre_audits["watchdog"] environment["DSV41_TOKENIZER_POLICY"] = canonical_json(prompt_policy["tokenizer"]) + environment["DSV41_MODEL_DESCRIPTOR"] = str(model_descriptor) + environment["DSV41_MODEL_DESCRIPTOR_IDENTITY"] = canonical_json(model_identity) + environment["DSV41_WATCHDOG_PIDFD"] = str(watchdog_descriptor) command = build_command(args, exporter, output) print("exec:", shlex.join(command), file=sys.stderr) verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") @@ -684,7 +846,10 @@ def main() -> int: label="candidate exporter", env=environment, check=False, + retained_fds=(model_descriptor, watchdog_descriptor), ) + verify_model_descriptor(model_descriptor, model_identity) + verify_watchdog_namespace_authority(watchdog_descriptor, watchdog_authority) if executed_identity != exporter_identity: raise PreflightError("candidate exporter execution identity differs from external approval") if result.returncode != 0: @@ -712,6 +877,8 @@ def main() -> int: repo=args.repo, busy_patterns=args.busy_pattern, ) + verify_watchdog_namespace_authority(watchdog_descriptor, watchdog_authority) + postflight_audit["watchdog"]["namespace_authority"] = watchdog_authority verify_approved_runtime_file_identities( runtime_identities, label="candidate exporter") post_accelerator = query_accelerator_attestation( @@ -774,6 +941,13 @@ def main() -> int: except (PreflightError, TraceError) as error: print(f"error: {error}", file=sys.stderr) return 1 + finally: + for descriptor in (watchdog_descriptor, model_descriptor): + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + print(f"error: cannot close retained descriptor {descriptor}: {error}", file=sys.stderr) if __name__ == "__main__": diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py index cc7a1994ed26..6b860c8f1f35 100644 --- a/tools/deepseek-v41-trace/run_matrix.py +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -53,6 +53,13 @@ ) +def approved_source_root(builder_policy: dict[str, object]) -> Path: + try: + return Path(str(builder_policy["source_root"])).expanduser().resolve(strict=True) + except (KeyError, OSError) as error: + raise PreflightError(f"prompt builder approved source root is invalid: {error}") from error + + def query_prompt_builder_runtime_build( builder: Path, builder_policy: dict[str, object]) -> dict[str, object]: @@ -133,11 +140,16 @@ def prepare_prompt( builder_policy, label="prompt builder") helper_identity = approved_containment_helper_identity( builder_policy, label="prompt builder") - expected_source = Path(builder_policy["source_root"]) / "tests" / "corpus" / corpus_name - if source_corpus != expected_source or sha256_file(source_corpus) != corpus_sha256 or ( + source_root_lexical = Path(builder_policy["source_root"]) + source_root_resolved = approved_source_root(builder_policy) + source_corpus_lexical = source_corpus + source_corpus_resolved = source_corpus_lexical.resolve(strict=True) + expected_source = ( + source_root_resolved / "tests" / "corpus" / corpus_name).resolve(strict=True) + if source_corpus_resolved != expected_source or sha256_file(source_corpus_resolved) != corpus_sha256 or ( sha256_file(corpus) != corpus_sha256): raise RuntimeError("prompt builder corpus path or bytes differ from external approval") - source_identity = file_identity(source_corpus) + source_identity = file_identity(source_corpus_resolved) corpus_identity = file_identity(corpus) verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") tokenizer = validate_tokenizer_policy(builder_policy["tokenizer"]) @@ -171,8 +183,8 @@ def prepare_prompt( raise RuntimeError("prompt builder execution identity differs from external approval") verify_approved_executable_identity(builder, builder_identity, label="prompt builder") verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") - if file_identity(source_corpus) != source_identity or file_identity(corpus) != corpus_identity or ( - sha256_file(source_corpus) != corpus_sha256) or sha256_file(corpus) != corpus_sha256: + if file_identity(source_corpus_resolved) != source_identity or file_identity(corpus) != corpus_identity or ( + sha256_file(source_corpus_resolved) != corpus_sha256) or sha256_file(corpus) != corpus_sha256: raise RuntimeError("prompt builder corpus changed during execution") if result.returncode != 0: raise RuntimeError(f"prompt builder failed: {result.stderr.strip()}") @@ -207,10 +219,14 @@ def prepare_prompt( builder_identity, runtime_identities, (helper_identity,)) record = { "format": "dsv41-prompt-provenance", - "version": 1, + "version": 2, "corpus_name": corpus_name, "corpus_sha256": corpus_sha256, - "corpus_path": str(source_corpus), + "corpus_path": str(source_corpus_resolved), + "corpus_lexical_path": str(source_corpus_lexical), + "corpus_resolved_path": str(source_corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), "model_sha256": MODEL_SHA256, "prompt_sha256": prompt_sha256, "prompt_byte_count": prompt_byte_count, @@ -399,7 +415,7 @@ def main() -> int: repo=repo, busy_patterns=args.busy_pattern, ) - if str(repo) != prompt_policy["source_root"] or args.candidate_revision != prompt_policy["revision"]: + if repo != approved_source_root(prompt_policy) or args.candidate_revision != prompt_policy["revision"]: raise PreflightError("matrix repository or revision differs from prompt builder approval") if output.exists() and any(output.iterdir()): raise PreflightError(f"matrix output directory is not empty: {output}") diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py index 669b8e2b5f6d..de0b7988d967 100644 --- a/tools/deepseek-v41-trace/trace_format.py +++ b/tools/deepseek-v41-trace/trace_format.py @@ -1852,6 +1852,7 @@ def run_approved_executable( expected_path: str, expected_sha256: str, label: str, + retained_fds: tuple[int, ...] = (), **kwargs: Any, ) -> tuple[subprocess.CompletedProcess[Any], ExecutableFileReceipt]: if sys.platform not in {"linux", "darwin", "win32"}: @@ -1862,6 +1863,16 @@ def run_approved_executable( "creationflags", "executable", "pass_fds", "preexec_fn", "process_group", "start_new_session"} if protected_controls & kwargs.keys(): raise TraceError(f"{label} execution parameters may not override immutable launch controls") + if sys.platform == "win32" and retained_fds: + raise TraceError(f"{label} retained descriptors are unsupported on Windows") + if any(type(descriptor) is not int or descriptor <= 2 for descriptor in retained_fds) or ( + len(set(retained_fds)) != len(retained_fds)): + raise TraceError(f"{label} retained descriptors are invalid") + try: + for retained_descriptor in retained_fds: + os.fstat(retained_descriptor) + except OSError as error: + raise TraceError(f"{label} retained descriptor is invalid: {error}") from error timeout = kwargs.pop("timeout", None) check = bool(kwargs.pop("check", False)) capture_output = bool(kwargs.pop("capture_output", False)) @@ -1948,6 +1959,7 @@ def run_approved_executable( retained_descriptors = ( descriptor, *(item[1] for item in runtime_files), + *retained_fds, ) launch = dict(kwargs) if sys.platform != "win32": @@ -4876,12 +4888,19 @@ def _validate_manifest(self) -> None: self.verifier.expected_prompt_builder_policy_id, policies=self.verifier.prompt_builder_policies, ) + source_root_lexical = Path(prompt_policy["source_root"]) + source_root_resolved = source_root_lexical.resolve(strict=False) + corpus_resolved = ( + source_root_resolved / "tests" / "corpus" / corpus_name).resolve(strict=False) provenance_checks = { "format": "dsv41-prompt-provenance", - "version": 1, + "version": 2, "corpus_name": corpus_name, "corpus_sha256": self.manifest["prompt"]["corpus_sha256"], - "corpus_path": f"{prompt_policy['source_root']}/tests/corpus/{corpus_name}", + "corpus_path": str(corpus_resolved), + "corpus_resolved_path": str(corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), "model_sha256": self.manifest["model"]["sha256"], "prompt_sha256": self.manifest["prompt"]["sha256"], "prompt_byte_count": self.manifest["prompt"]["byte_count"], @@ -4893,6 +4912,11 @@ def _validate_manifest(self) -> None: for key, value in provenance_checks.items(): if provenance_record.get(key) != value: raise TraceError(f"prompt provenance {key} mismatch") + try: + if Path(str(provenance_record.get("corpus_lexical_path"))).resolve(strict=False) != corpus_resolved: + raise TraceError("prompt provenance corpus lexical path resolves outside the approved source") + except OSError as error: + raise TraceError(f"prompt provenance corpus lexical path is invalid: {error}") from error expected_prompt = approved_prompt_record( prompt_policy, corpus_name=corpus_name, @@ -4938,6 +4962,7 @@ def _validate_manifest(self) -> None: _require_exact_keys( provenance_record, set(provenance_checks) | set(builder_checks) | { + "corpus_lexical_path", "builder_runtime_build", "builder_runtime_build_sha256", "builder_install_trust", "builder_install_trust_sha256"}, "prompt provenance", @@ -5171,6 +5196,8 @@ def _validate_manifest(self) -> None: "expert_cache_slots", "expert_cache_bytes", "tokenizer", + "model_file_identity", + "watchdog_namespace", "deepseek41", }, "llama.cpp config", @@ -5192,6 +5219,87 @@ def _validate_manifest(self) -> None: raise TraceError("llama.cpp trace inference configuration is invalid") if validate_tokenizer_policy(config.get("tokenizer")) != prompt_policy["tokenizer"]: raise TraceError("llama.cpp tokenizer policy differs from external prompt approval") + model_file_identity = config.get("model_file_identity") + if not isinstance(model_file_identity, dict): + raise TraceError("llama.cpp model file identity is invalid") + _require_exact_keys( + model_file_identity, + { + "format", + "version", + "path", + "device", + "inode", + "owner_uid", + "owner_gid", + "mode", + "link_count", + "byte_count", + "modified_ns", + "changed_ns", + "status_flags", + "source_descriptor_flags", + "target_descriptor_flags", + "sha256", + }, + "llama.cpp model file identity", + ) + if model_file_identity["format"] != "dsv41-model-file-identity" or ( + model_file_identity["version"] != 1) or ( + model_file_identity["path"] != self.manifest["model"]["path"]) or ( + model_file_identity["byte_count"] != self.manifest["model"]["byte_count"]) or ( + model_file_identity["sha256"] != self.manifest["model"]["sha256"]): + raise TraceError("llama.cpp model file identity does not match the manifest model") + for key in ( + "device", "inode", "owner_uid", "owner_gid", "mode", "link_count", + "byte_count", "modified_ns", "changed_ns", "status_flags", + "source_descriptor_flags", "target_descriptor_flags"): + if type(model_file_identity[key]) is not int or model_file_identity[key] < 0: + raise TraceError(f"llama.cpp model file identity {key} is invalid") + if model_file_identity["link_count"] < 1 or model_file_identity["mode"] > 0o7777 or ( + model_file_identity["source_descriptor_flags"] != 1) or ( + model_file_identity["target_descriptor_flags"] != 0): + raise TraceError("llama.cpp model descriptor policy is invalid") + watchdog_namespace = config.get("watchdog_namespace") + if not isinstance(watchdog_namespace, dict): + raise TraceError("llama.cpp watchdog namespace binding is invalid") + _require_exact_keys( + watchdog_namespace, + { + "format", + "version", + "authority", + "host_watchdog_pid", + "host_watchdog_process_group_id", + "host_watchdog_start_time_ticks", + "local_pid", + "local_parent_pid", + "local_process_group_id", + "local_session_id", + "namespace_pids", + "private_procfs", + }, + "llama.cpp watchdog namespace binding", + ) + if watchdog_namespace["format"] != "dsv41-watchdog-namespace-binding" or ( + watchdog_namespace["version"] != 1) or ( + watchdog_namespace["authority"] != "inherited-pidfd") or ( + watchdog_namespace["private_procfs"] is not True): + raise TraceError("llama.cpp watchdog namespace binding policy is invalid") + for key in ( + "host_watchdog_pid", "host_watchdog_process_group_id", + "host_watchdog_start_time_ticks", "local_pid", + "local_process_group_id", "local_session_id"): + if type(watchdog_namespace[key]) is not int or watchdog_namespace[key] <= 1: + raise TraceError(f"llama.cpp watchdog namespace {key} is invalid") + if watchdog_namespace["local_parent_pid"] != 1 or ( + watchdog_namespace["local_pid"] != 2) or ( + watchdog_namespace["local_pid"] != watchdog_namespace["local_process_group_id"]) or ( + watchdog_namespace["local_pid"] != watchdog_namespace["local_session_id"]) or ( + not isinstance(watchdog_namespace["namespace_pids"], list)) or ( + not watchdog_namespace["namespace_pids"]) or ( + watchdog_namespace["namespace_pids"][-1] != watchdog_namespace["local_pid"]): + raise TraceError("llama.cpp watchdog namespace-local identity is invalid") if self.manifest["runtime"] == "ds4": _require_exact_keys( config, @@ -5445,6 +5553,7 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: "audit_fd", "audit_sha256", "audit", + "namespace_authority", ) _require_exact_keys(record["data"], set(required), f"{phase} watchdog audit evidence") data = record["data"] @@ -5506,6 +5615,48 @@ def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: raise TraceError(f"{phase} watchdog audit command SHA-256 is invalid") if type(data["heartbeat_unix"]) is not int or data["heartbeat_unix"] <= 0: raise TraceError(f"{phase} watchdog audit heartbeat timestamp is invalid") + namespace_authority = data["namespace_authority"] + if not isinstance(namespace_authority, dict): + raise TraceError(f"{phase} watchdog namespace authority is invalid") + _require_exact_keys( + namespace_authority, + { + "format", + "version", + "mechanism", + "descriptor", + "host_procfs_root", + "watchdog_pid", + "watchdog_process_group_id", + "watchdog_start_time_ticks", + "watchdog_executable_path", + "watchdog_command_sha256", + "guardian_pid", + "child_pid", + "child_process_group_id", + }, + f"{phase} watchdog namespace authority", + ) + if namespace_authority["format"] != "dsv41-watchdog-namespace-authority" or ( + namespace_authority["version"] != 1) or ( + namespace_authority["mechanism"] != "inherited-pidfd") or ( + namespace_authority["host_procfs_root"] != "/proc"): + raise TraceError(f"{phase} watchdog namespace authority policy is invalid") + authority_checks = { + "watchdog_pid": data["watchdog_pid"], + "watchdog_start_time_ticks": data["watchdog_start_time_ticks"], + "watchdog_executable_path": data["watchdog_executable_path"], + "watchdog_command_sha256": data["watchdog_command_sha256"], + "guardian_pid": data["guardian_pid"], + "child_pid": data["child_pid"], + "child_process_group_id": data["child_process_group_id"], + } + for key, value in authority_checks.items(): + if namespace_authority.get(key) != value: + raise TraceError(f"{phase} watchdog namespace authority {key} mismatch") + for key in ("descriptor", "watchdog_process_group_id"): + if type(namespace_authority[key]) is not int or namespace_authority[key] <= 2: + raise TraceError(f"{phase} watchdog namespace authority {key} is invalid") max_age = data.get("max_heartbeat_age_seconds") if not isinstance(max_age, (int, float)) or isinstance(max_age, bool) or ( max_age <= 0 or max_age > 30): From 170796fe3cb9e1efa4f43480b5447b8ba8d102e1 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 05:31:19 -0700 Subject: [PATCH 51/56] ci : run DeepSeek V4.1 trace test on Linux Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/server.yml | 126 ++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 77fe7dbd3aa7..cb7b5ca83e42 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -98,8 +98,14 @@ jobs: id: cmake_build run: | cmake -B build \ - -DGGML_SCHED_NO_REALLOC=ON - cmake --build build --config Release -j $(nproc) --target llama-server + -DGGML_SCHED_NO_REALLOC=ON \ + -DLLAMA_OPENSSL=OFF + cmake --build build --config Release -j $(nproc) --target \ + llama-server \ + llama-deepseek-v41-trace \ + llama-deepseek-v41-containment-helper \ + test-deepseek41-trace-manifest \ + test-deepseek41-trace-injected - name: ccache-buckets-save if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} @@ -120,6 +126,122 @@ jobs: python-version: '3.11' pip-install: -r tools/server/tests/requirements.txt + - name: DeepSeek V4.1 trace CTest + id: deepseek_v41_trace_ctest + run: | + set -o pipefail + ctest --test-dir build --show-only=json-v1 -R '^test-deepseek41-trace$' > build/test-deepseek41-trace-ctest-metadata.json + python3 - <<'PY' + import json + from pathlib import Path + + metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) + tests = metadata.get("tests", []) + if len(tests) != 1 or tests[0].get("name") != "test-deepseek41-trace": + raise SystemExit(f"expected exactly one test-deepseek41-trace selector match, got {len(tests)}") + properties = { + item["name"]: item["value"] + for item in tests[0].get("properties", []) + } + environment = properties.get("ENVIRONMENT", []) + if isinstance(environment, str): + environment = environment.split(";") + required = ( + "DSV41_NATIVE_TRACE_BINARY=", + "DSV41_NATIVE_CONTAINMENT_HELPER=", + "DSV41_NATIVE_MANIFEST_BINARY=", + "DSV41_NATIVE_INJECT_LIBRARY=", + ) + missing = [ + name + for name in required + if not any(value.startswith(name) for value in environment) + ] + if missing: + raise SystemExit(f"test-deepseek41-trace environment is incomplete: {missing}") + PY + : > build/test-deepseek41-trace-ctest.log + ctest --test-dir build \ + -R '^test-deepseek41-trace$' \ + --output-on-failure \ + --verbose \ + --no-tests=error \ + --output-junit test-deepseek41-trace-ctest.xml \ + 2>&1 | tee build/test-deepseek41-trace-ctest.log + python3 - <<'PY' + from pathlib import Path + from xml.etree import ElementTree + + report = ElementTree.parse("build/test-deepseek41-trace-ctest.xml").getroot() + expected = { + "tests": 1, + "failures": 0, + "disabled": 0, + "skipped": 0, + } + actual = { + name: int(report.attrib.get(name, -1)) + for name in expected + } + if actual != expected: + raise SystemExit(f"test-deepseek41-trace JUnit result is invalid: {actual}") + log = Path("build/test-deepseek41-trace-ctest.log").read_text(errors="replace") + if "***Skipped" in log or "***Not Run" in log or "OK (skipped=" in log: + raise SystemExit("test-deepseek41-trace contained a skipped or not-run test") + PY + python3 - <<'PY' + import json + import os + import subprocess + from pathlib import Path + + metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) + test = metadata["tests"][0] + properties = { + item["name"]: item["value"] + for item in test.get("properties", []) + } + environment = properties["ENVIRONMENT"] + if isinstance(environment, str): + environment = environment.split(";") + child_environment = dict(os.environ) + for value in environment: + name, setting = value.split("=", 1) + child_environment[name] = setting + test_name = "TraceFormatTests.test_native_watchdog_validation_crosses_private_pid_namespace" + result = subprocess.run( + [*test["command"], test_name, "--verbose"], + cwd=properties["WORKING_DIRECTORY"], + env=child_environment, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + output = result.stdout + Path("build/test-deepseek41-trace-linux-pidns.log").write_text(output, encoding="utf-8") + print(output, end="") + if result.returncode != 0: + raise SystemExit(result.returncode) + if output.count("Ran 1 test") != 1 or test_name not in output: + raise SystemExit("private PID namespace/pidfd test count is not exactly one") + if "skipped" in output.lower() or not output.rstrip().endswith("OK"): + raise SystemExit("private PID namespace/pidfd test was skipped or did not pass") + PY + + - name: Upload DeepSeek V4.1 trace CTest evidence + if: ${{ always() }} + uses: actions/upload-artifact@v6 + with: + name: deepseek-v41-trace-ctest-${{ github.run_id }}-${{ github.run_attempt }} + path: | + build/test-deepseek41-trace-ctest-metadata.json + build/test-deepseek41-trace-ctest.log + build/test-deepseek41-trace-ctest.xml + build/test-deepseek41-trace-linux-pidns.log + if-no-files-found: error + retention-days: 7 + - name: Tests id: server_integration_tests run: | From 4a12c80103b6ac2b12fe08bfda621013b080ece8 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 05:49:14 -0700 Subject: [PATCH 52/56] ci : prove DeepSeek trace pidns runs once Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/server.yml | 146 ++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 28 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index cb7b5ca83e42..7be64736fb23 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -160,8 +160,77 @@ jobs: if missing: raise SystemExit(f"test-deepseek41-trace environment is incomplete: {missing}") PY + rm -rf build/deepseek-v41-trace-hook build/deepseek-v41-trace-marker + install -d -m 0700 build/deepseek-v41-trace-hook build/deepseek-v41-trace-marker + cat > build/deepseek-v41-trace-hook/sitecustomize.py <<'PY' + import hashlib + import json + import os + import time + import unittest + from pathlib import Path + + _TEST_CLASS = "TraceFormatTests" + _TEST_METHOD = "test_native_watchdog_validation_crosses_private_pid_namespace" + _TEST_ID = f"__main__.{_TEST_CLASS}.{_TEST_METHOD}" + _SOURCE = Path(os.environ["DSV41_CI_TEST_SOURCE"]).resolve(strict=True) + _MARKER_DIR = Path(os.environ["DSV41_CI_MARKER_DIR"]).resolve(strict=True) + _BINDINGS = ( + "DSV41_NATIVE_TRACE_BINARY", + "DSV41_NATIVE_CONTAINMENT_HELPER", + "DSV41_NATIVE_MANIFEST_BINARY", + "DSV41_NATIVE_INJECT_LIBRARY", + ) + _ORIGINAL_CALL_TEST_METHOD = unittest.TestCase._callTestMethod + + def _write_marker(path, payload): + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii")) + os.fsync(descriptor) + finally: + os.close(descriptor) + + def _call_test_method(self, method): + function = getattr(method, "__func__", method) + code = getattr(function, "__code__", None) + source = Path(code.co_filename).resolve(strict=True) if code is not None else None + if self.__class__.__name__ == _TEST_CLASS and method.__name__ == _TEST_METHOD and source == _SOURCE: + bindings = {} + for name in _BINDINGS: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"missing {name} binding") + bindings[name] = str(Path(value).resolve(strict=True)) + payload = { + "bindings": bindings, + "event": "test_method_entry", + "first_line": code.co_firstlineno, + "function": _TEST_METHOD, + "pid": os.getpid(), + "ppid": os.getppid(), + "source": str(source), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "test_id": self.id(), + "timestamp_monotonic_ns": time.monotonic_ns(), + } + try: + _write_marker(_MARKER_DIR / "entry.json", payload) + except FileExistsError: + duplicate = _MARKER_DIR / f"duplicate-{os.getpid()}-{time.monotonic_ns()}.json" + _write_marker(duplicate, payload) + raise RuntimeError(f"{_TEST_ID} entered more than once") + return _ORIGINAL_CALL_TEST_METHOD(self, method) + + unittest.TestCase._callTestMethod = _call_test_method + PY + chmod 0444 build/deepseek-v41-trace-hook/sitecustomize.py + cmake -E sha256sum build/deepseek-v41-trace-hook/sitecustomize.py > build/deepseek-v41-trace-hook/sitecustomize.sha256 : > build/test-deepseek41-trace-ctest.log - ctest --test-dir build \ + PYTHONPATH="$PWD/build/deepseek-v41-trace-hook${PYTHONPATH:+:$PYTHONPATH}" \ + DSV41_CI_MARKER_DIR="$PWD/build/deepseek-v41-trace-marker" \ + DSV41_CI_TEST_SOURCE="$PWD/tests/test-deepseek41-trace.py" \ + ctest --test-dir build \ -R '^test-deepseek41-trace$' \ --output-on-failure \ --verbose \ @@ -169,6 +238,8 @@ jobs: --output-junit test-deepseek41-trace-ctest.xml \ 2>&1 | tee build/test-deepseek41-trace-ctest.log python3 - <<'PY' + import hashlib + import json from pathlib import Path from xml.etree import ElementTree @@ -188,12 +259,6 @@ jobs: log = Path("build/test-deepseek41-trace-ctest.log").read_text(errors="replace") if "***Skipped" in log or "***Not Run" in log or "OK (skipped=" in log: raise SystemExit("test-deepseek41-trace contained a skipped or not-run test") - PY - python3 - <<'PY' - import json - import os - import subprocess - from pathlib import Path metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) test = metadata["tests"][0] @@ -204,29 +269,52 @@ jobs: environment = properties["ENVIRONMENT"] if isinstance(environment, str): environment = environment.split(";") - child_environment = dict(os.environ) + expected_bindings = {} for value in environment: name, setting = value.split("=", 1) - child_environment[name] = setting + if name.startswith("DSV41_NATIVE_"): + expected_bindings[name] = str(Path(setting).resolve(strict=True)) + if set(expected_bindings) != { + "DSV41_NATIVE_TRACE_BINARY", + "DSV41_NATIVE_CONTAINMENT_HELPER", + "DSV41_NATIVE_MANIFEST_BINARY", + "DSV41_NATIVE_INJECT_LIBRARY", + }: + raise SystemExit("test-deepseek41-trace native bindings are invalid") test_name = "TraceFormatTests.test_native_watchdog_validation_crosses_private_pid_namespace" - result = subprocess.run( - [*test["command"], test_name, "--verbose"], - cwd=properties["WORKING_DIRECTORY"], - env=child_environment, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - output = result.stdout - Path("build/test-deepseek41-trace-linux-pidns.log").write_text(output, encoding="utf-8") - print(output, end="") - if result.returncode != 0: - raise SystemExit(result.returncode) - if output.count("Ran 1 test") != 1 or test_name not in output: - raise SystemExit("private PID namespace/pidfd test count is not exactly one") - if "skipped" in output.lower() or not output.rstrip().endswith("OK"): - raise SystemExit("private PID namespace/pidfd test was skipped or did not pass") + marker_dir = Path("build/deepseek-v41-trace-marker") + entries = list(marker_dir.glob("entry.json")) + duplicates = list(marker_dir.glob("duplicate-*.json")) + unexpected = [ + path + for path in marker_dir.iterdir() + if path not in entries and path not in duplicates + ] + if len(entries) != 1 or duplicates or unexpected: + raise SystemExit( + f"private PID namespace/pidfd entry count is invalid: " + f"entries={len(entries)} duplicates={len(duplicates)} unexpected={len(unexpected)}") + marker = json.loads(entries[0].read_text()) + expected_source = Path("tests/test-deepseek41-trace.py").resolve(strict=True) + expected_marker = { + "bindings": expected_bindings, + "event": "test_method_entry", + "function": test_name.split(".", 1)[1], + "source": str(expected_source), + "source_sha256": hashlib.sha256(expected_source.read_bytes()).hexdigest(), + "test_id": f"__main__.{test_name}", + } + for name, value in expected_marker.items(): + if marker.get(name) != value: + raise SystemExit(f"private PID namespace/pidfd marker {name} is invalid") + if not isinstance(marker.get("first_line"), int) or marker["first_line"] <= 0: + raise SystemExit("private PID namespace/pidfd marker first_line is invalid") + if not isinstance(marker.get("pid"), int) or marker["pid"] <= 1: + raise SystemExit("private PID namespace/pidfd marker pid is invalid") + if not isinstance(marker.get("ppid"), int) or marker["ppid"] <= 0: + raise SystemExit("private PID namespace/pidfd marker ppid is invalid") + if not isinstance(marker.get("timestamp_monotonic_ns"), int) or marker["timestamp_monotonic_ns"] <= 0: + raise SystemExit("private PID namespace/pidfd marker timestamp is invalid") PY - name: Upload DeepSeek V4.1 trace CTest evidence @@ -238,7 +326,9 @@ jobs: build/test-deepseek41-trace-ctest-metadata.json build/test-deepseek41-trace-ctest.log build/test-deepseek41-trace-ctest.xml - build/test-deepseek41-trace-linux-pidns.log + build/deepseek-v41-trace-hook/sitecustomize.py + build/deepseek-v41-trace-hook/sitecustomize.sha256 + build/deepseek-v41-trace-marker/ if-no-files-found: error retention-days: 7 From b453d01b8c16faf7ecc3a367d2680c1a22c42831 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 06:09:56 -0700 Subject: [PATCH 53/56] ci : bind DeepSeek trace to Python 3.11 Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/server.yml | 87 +++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 7be64736fb23..2e91704ed11e 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -94,12 +94,20 @@ jobs: folder: llama.cpp hf_bucket: ggml-org/cache + - name: Python setup + id: setup_python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + pip-install: -r tools/server/tests/requirements.txt + - name: Build id: cmake_build run: | cmake -B build \ -DGGML_SCHED_NO_REALLOC=ON \ - -DLLAMA_OPENSSL=OFF + -DLLAMA_OPENSSL=OFF \ + -DPython3_EXECUTABLE="$(command -v python3)" cmake --build build --config Release -j $(nproc) --target \ llama-server \ llama-deepseek-v41-trace \ @@ -119,13 +127,6 @@ jobs: hf_bucket: ggml-org/cache save: true - - name: Python setup - id: setup_python - uses: actions/setup-python@v6 - with: - python-version: '3.11' - pip-install: -r tools/server/tests/requirements.txt - - name: DeepSeek V4.1 trace CTest id: deepseek_v41_trace_ctest run: | @@ -133,12 +134,40 @@ jobs: ctest --test-dir build --show-only=json-v1 -R '^test-deepseek41-trace$' > build/test-deepseek41-trace-ctest-metadata.json python3 - <<'PY' import json + import subprocess + import sys from pathlib import Path metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) tests = metadata.get("tests", []) if len(tests) != 1 or tests[0].get("name") != "test-deepseek41-trace": raise SystemExit(f"expected exactly one test-deepseek41-trace selector match, got {len(tests)}") + command = tests[0].get("command", []) + if not command or not isinstance(command[0], str) or not Path(command[0]).is_absolute(): + raise SystemExit("test-deepseek41-trace interpreter is absent or relative") + registered_interpreter = Path(command[0]).resolve(strict=True) + selected_interpreter = Path(sys.executable).resolve(strict=True) + if registered_interpreter != selected_interpreter: + raise SystemExit( + f"test-deepseek41-trace interpreter mismatch: " + f"registered={registered_interpreter} selected={selected_interpreter}") + registered_version = subprocess.check_output( + [str(registered_interpreter), "-c", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"], + text=True, + ).strip() + if registered_version != "3.11" or sys.version_info[:2] != (3, 11): + raise SystemExit( + f"test-deepseek41-trace requires Python 3.11: " + f"registered={registered_version} selected={sys.version_info.major}.{sys.version_info.minor}") + Path("build/test-deepseek41-trace-python.json").write_text( + json.dumps({ + "registered_interpreter": str(registered_interpreter), + "registered_version": registered_version, + "selected_interpreter": str(selected_interpreter), + "selected_version": f"{sys.version_info.major}.{sys.version_info.minor}", + }, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) properties = { item["name"]: item["value"] for item in tests[0].get("properties", []) @@ -182,6 +211,7 @@ jobs: "DSV41_NATIVE_INJECT_LIBRARY", ) _ORIGINAL_CALL_TEST_METHOD = unittest.TestCase._callTestMethod + _ORIGINAL_ADD_SKIP = unittest.TextTestResult.addSkip def _write_marker(path, payload): descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) @@ -222,7 +252,25 @@ jobs: raise RuntimeError(f"{_TEST_ID} entered more than once") return _ORIGINAL_CALL_TEST_METHOD(self, method) + def _add_skip(self, test, reason): + method = getattr(test, getattr(test, "_testMethodName", ""), None) + function = getattr(method, "__func__", method) + code = getattr(function, "__code__", None) + source = Path(code.co_filename).resolve(strict=True) if code is not None else None + test_id = test.id() + payload = { + "event": "test_skip", + "reason": reason, + "source": str(source) if source is not None else None, + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest() if source is not None else None, + "test_id": test_id, + } + digest = hashlib.sha256(test_id.encode("ascii")).hexdigest() + _write_marker(_MARKER_DIR / f"skip-{digest}.json", payload) + return _ORIGINAL_ADD_SKIP(self, test, reason) + unittest.TestCase._callTestMethod = _call_test_method + unittest.TextTestResult.addSkip = _add_skip PY chmod 0444 build/deepseek-v41-trace-hook/sitecustomize.py cmake -E sha256sum build/deepseek-v41-trace-hook/sitecustomize.py > build/deepseek-v41-trace-hook/sitecustomize.sha256 @@ -240,6 +288,7 @@ jobs: python3 - <<'PY' import hashlib import json + import sys from pathlib import Path from xml.etree import ElementTree @@ -257,8 +306,8 @@ jobs: if actual != expected: raise SystemExit(f"test-deepseek41-trace JUnit result is invalid: {actual}") log = Path("build/test-deepseek41-trace-ctest.log").read_text(errors="replace") - if "***Skipped" in log or "***Not Run" in log or "OK (skipped=" in log: - raise SystemExit("test-deepseek41-trace contained a skipped or not-run test") + if "***Skipped" in log or "***Not Run" in log: + raise SystemExit("test-deepseek41-trace was skipped or not run") metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) test = metadata["tests"][0] @@ -285,10 +334,11 @@ jobs: marker_dir = Path("build/deepseek-v41-trace-marker") entries = list(marker_dir.glob("entry.json")) duplicates = list(marker_dir.glob("duplicate-*.json")) + skips = list(marker_dir.glob("skip-*.json")) unexpected = [ path for path in marker_dir.iterdir() - if path not in entries and path not in duplicates + if path not in entries and path not in duplicates and path not in skips ] if len(entries) != 1 or duplicates or unexpected: raise SystemExit( @@ -315,6 +365,20 @@ jobs: raise SystemExit("private PID namespace/pidfd marker ppid is invalid") if not isinstance(marker.get("timestamp_monotonic_ns"), int) or marker["timestamp_monotonic_ns"] <= 0: raise SystemExit("private PID namespace/pidfd marker timestamp is invalid") + if sys.platform != "linux": + raise SystemExit(f"test-deepseek41-trace requires Linux, got {sys.platform}") + if len(skips) != 1: + raise SystemExit(f"test-deepseek41-trace approved skip count is invalid: {len(skips)}") + approved_skip = json.loads(skips[0].read_text()) + expected_skip = { + "event": "test_skip", + "reason": "Linux uses subreaper and pidfd containment", + "source": str(expected_source), + "source_sha256": hashlib.sha256(expected_source.read_bytes()).hexdigest(), + "test_id": "__main__.TraceFormatTests.test_unproven_posix_containment_fails_closed_before_setsid_escape", + } + if approved_skip != expected_skip: + raise SystemExit("test-deepseek41-trace skip identity is invalid") PY - name: Upload DeepSeek V4.1 trace CTest evidence @@ -324,6 +388,7 @@ jobs: name: deepseek-v41-trace-ctest-${{ github.run_id }}-${{ github.run_attempt }} path: | build/test-deepseek41-trace-ctest-metadata.json + build/test-deepseek41-trace-python.json build/test-deepseek41-trace-ctest.log build/test-deepseek41-trace-ctest.xml build/deepseek-v41-trace-hook/sitecustomize.py From fc88e35f1c44509315d6a8bf23ac05dee2c02268 Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 09:59:36 -0700 Subject: [PATCH 54/56] ci : normalize DeepSeek trace runner identity Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/server.yml | 883 +++++++++++++++++++++++++++++------ 1 file changed, 729 insertions(+), 154 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 2e91704ed11e..79ce24d52a52 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -131,14 +131,46 @@ jobs: id: deepseek_v41_trace_ctest run: | set -o pipefail - ctest --test-dir build --show-only=json-v1 -R '^test-deepseek41-trace$' > build/test-deepseek41-trace-ctest-metadata.json + ci_root="$PWD/build/deepseek-v41-trace-ci" + rm -rf "$ci_root" + install -d -m 0700 \ + "$ci_root" \ + "$ci_root/tmp" \ + "$ci_root/hook" \ + "$ci_root/marker" \ + "$ci_root/log" \ + "$ci_root/evidence" + runner_uid="$(id -u)" + runner_gid="$(id -g)" + runner_cap_bnd="$(awk '/^CapBnd:/ { print $2 }' /proc/self/status)" + selected_interpreter="$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.executable).resolve(strict=True))')" + ctest_path="$(command -v ctest)" + if [[ "$runner_uid" == "0" || "$runner_gid" == "0" || ! "$runner_cap_bnd" =~ ^[0-9A-Fa-f]+$ || "$selected_interpreter" != /* || "$ctest_path" != /* ]]; then + echo "hosted runner identity or executable path is invalid" >&2 + exit 1 + fi + export DSV41_CI_ROOT="$ci_root" + export DSV41_CI_RUNNER_UID="$runner_uid" + export DSV41_CI_RUNNER_GID="$runner_gid" + export DSV41_CI_RUNNER_CAP_BND="$runner_cap_bnd" + export DSV41_CI_SELECTED_INTERPRETER="$selected_interpreter" + export DSV41_CI_CTEST="$(realpath "$ctest_path")" + ctest --test-dir build --show-only=json-v1 -R '^test-deepseek41-trace$' > "$ci_root/evidence/test-deepseek41-trace-ctest-metadata.json" python3 - <<'PY' + import ast + import hashlib import json + import os import subprocess import sys from pathlib import Path - metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) + root = Path(os.environ["DSV41_CI_ROOT"]).resolve(strict=True) + evidence = root / "evidence" + workspace = Path.cwd().resolve(strict=True) + build = (workspace / "build").resolve(strict=True) + metadata_path = evidence / "test-deepseek41-trace-ctest-metadata.json" + metadata = json.loads(metadata_path.read_text()) tests = metadata.get("tests", []) if len(tests) != 1 or tests[0].get("name") != "test-deepseek41-trace": raise SystemExit(f"expected exactly one test-deepseek41-trace selector match, got {len(tests)}") @@ -159,7 +191,7 @@ jobs: raise SystemExit( f"test-deepseek41-trace requires Python 3.11: " f"registered={registered_version} selected={sys.version_info.major}.{sys.version_info.minor}") - Path("build/test-deepseek41-trace-python.json").write_text( + (evidence / "test-deepseek41-trace-python.json").write_text( json.dumps({ "registered_interpreter": str(registered_interpreter), "registered_version": registered_version, @@ -188,198 +220,747 @@ jobs: ] if missing: raise SystemExit(f"test-deepseek41-trace environment is incomplete: {missing}") + source = workspace / "tools/deepseek-v41-trace/trace_format.py" + parsed = ast.parse(source.read_text(encoding="ascii")) + forbidden = None + for node in parsed.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "FORBIDDEN_LOADER_ENVIRONMENT" + for target in node.targets): + forbidden = ast.literal_eval(node.value) + break + expected_forbidden = ( + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_IMAGE_SUFFIX", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "DYLD_ROOT_PATH", + "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", + "GGML_BACKEND_PATH", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + ) + if forbidden != expected_forbidden: + raise SystemExit("trace forbidden loader environment changed") + forbidden_path = evidence / "forbidden-loader-environment.json" + forbidden_path.write_text( + json.dumps(list(forbidden), separators=(",", ":")) + "\n", + encoding="ascii", + ) + config = { + "ctest": str(Path(os.environ["DSV41_CI_CTEST"]).resolve(strict=True)), + "forbidden_path": str(forbidden_path), + "forbidden_sha256": hashlib.sha256(forbidden_path.read_bytes()).hexdigest(), + "environment_allowlist": sorted(( + "DSV41_CI_CONFIG", + "DSV41_CI_CONFIG_SHA256", + "DSV41_CI_GATE", + "DSV41_CI_GATE_SHA256", + "DSV41_CI_HOOK_SHA256", + "HOME", + "LANG", + "LC_ALL", + "PATH", + "PYTHONDONTWRITEBYTECODE", + "PYTHONCOERCECLOCALE", + "PYTHONPATH", + "PYTHONUTF8", + "TMPDIR", + )), + "paths": { + "build": str(build), + "evidence": str(evidence), + "hook": str(root / "hook"), + "log": str(root / "log"), + "marker": str(root / "marker"), + "tmp": str(root / "tmp"), + "workspace": str(workspace), + }, + "registered_interpreter": str(registered_interpreter), + "runner_cap_bnd": int(os.environ["DSV41_CI_RUNNER_CAP_BND"], 16), + "runner_gid": int(os.environ["DSV41_CI_RUNNER_GID"]), + "runner_uid": int(os.environ["DSV41_CI_RUNNER_UID"]), + "selected_interpreter": str(selected_interpreter), + "source": str((workspace / "tests/test-deepseek41-trace.py").resolve(strict=True)), + } + (evidence / "fixture-config.json").write_text( + json.dumps(config, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) PY - rm -rf build/deepseek-v41-trace-hook build/deepseek-v41-trace-marker - install -d -m 0700 build/deepseek-v41-trace-hook build/deepseek-v41-trace-marker - cat > build/deepseek-v41-trace-hook/sitecustomize.py <<'PY' + cat > "$ci_root/evidence/fixture_gate.py" <<'PY' import hashlib import json import os + import stat + import sys + import tempfile import time - import unittest from pathlib import Path - _TEST_CLASS = "TraceFormatTests" - _TEST_METHOD = "test_native_watchdog_validation_crosses_private_pid_namespace" - _TEST_ID = f"__main__.{_TEST_CLASS}.{_TEST_METHOD}" - _SOURCE = Path(os.environ["DSV41_CI_TEST_SOURCE"]).resolve(strict=True) - _MARKER_DIR = Path(os.environ["DSV41_CI_MARKER_DIR"]).resolve(strict=True) - _BINDINGS = ( - "DSV41_NATIVE_TRACE_BINARY", - "DSV41_NATIVE_CONTAINMENT_HELPER", - "DSV41_NATIVE_MANIFEST_BINARY", - "DSV41_NATIVE_INJECT_LIBRARY", + FORBIDDEN = ( + "DYLD_FALLBACK_FRAMEWORK_PATH", "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", "DYLD_IMAGE_SUFFIX", "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", "DYLD_ROOT_PATH", "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", "GGML_BACKEND_PATH", "LD_AUDIT", + "LD_LIBRARY_PATH", "LD_PRELOAD", ) - _ORIGINAL_CALL_TEST_METHOD = unittest.TestCase._callTestMethod - _ORIGINAL_ADD_SKIP = unittest.TextTestResult.addSkip - def _write_marker(path, payload): - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + def bytes_for(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + def write_exclusive(path, value): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + data = bytes_for(value) try: - os.write(descriptor, (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii")) + if os.write(descriptor, data) != len(data): + raise RuntimeError(f"short write for {path}") os.fsync(descriptor) finally: os.close(descriptor) + return hashlib.sha256(data).hexdigest() + + def verify_digest(path, expected): + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + raise RuntimeError(f"SHA-256 mismatch for {path}") + return actual + + def read_status(text=None): + text = Path("/proc/self/status").read_text(encoding="ascii") if text is None else text + fields = {} + for line in text.splitlines(): + name, separator, value = line.partition(":") + if separator: + fields[name] = value.strip() + required = ("Uid", "Gid", "Groups", "CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb", "NoNewPrivs") + missing = [name for name in required if name not in fields] + if missing: + raise RuntimeError(f"/proc/self/status is incomplete: {missing}") + return { + "cap_amb": int(fields["CapAmb"], 16), + "cap_bnd": int(fields["CapBnd"], 16), + "cap_eff": int(fields["CapEff"], 16), + "cap_inh": int(fields["CapInh"], 16), + "cap_prm": int(fields["CapPrm"], 16), + "gid": [int(value) for value in fields["Gid"].split()], + "groups": [int(value) for value in fields["Groups"].split()], + "no_new_privs": int(fields["NoNewPrivs"]), + "uid": [int(value) for value in fields["Uid"].split()], + } + + def current_state(): + state = read_status() + state.update({ + "os_egid": os.getegid(), "os_euid": os.geteuid(), + "os_gid": os.getgid(), "os_groups": os.getgroups(), "os_uid": os.getuid(), + }) + return state + + def validate_state(state, uid, gid, cap_bnd): + if uid <= 0 or gid <= 0: + raise RuntimeError("runner identity is privileged") + if state["uid"] != [uid] * 4 or state["gid"] != [gid] * 4: + raise RuntimeError("live UID or GID differs") + if (state["os_uid"], state["os_euid"], state["os_gid"], state["os_egid"]) != (uid, uid, gid, gid): + raise RuntimeError("runtime UID or GID differs") + if state["groups"] or state["os_groups"]: + raise RuntimeError("supplementary groups are not empty") + for name in ("cap_inh", "cap_prm", "cap_eff", "cap_amb"): + if state[name] != 0: + raise RuntimeError(f"{name} is not zero") + if state["cap_bnd"] != cap_bnd: + raise RuntimeError("CapBnd changed") + if state["no_new_privs"] != 1: + raise RuntimeError("NoNewPrivs is not one") + + def validate_interpreter_values(registered, selected, executable, registered_version, selected_version): + if not all(Path(value).is_absolute() for value in (registered, selected, executable)): + raise RuntimeError("Python path is not absolute") + if registered != selected or registered != executable: + raise RuntimeError("Python identity differs") + if registered_version != "3.11" or selected_version != "3.11": + raise RuntimeError("Python is not version 3.11") + + def validate_interpreter(config): + registered = str(Path(config["registered_interpreter"]).resolve(strict=True)) + selected = str(Path(config["selected_interpreter"]).resolve(strict=True)) + executable = str(Path(sys.executable).resolve(strict=True)) + version = f"{sys.version_info.major}.{sys.version_info.minor}" + validate_interpreter_values(registered, selected, executable, version, version) + return {"executable": executable, "registered": registered, "selected": selected, "version": version} + + def load_forbidden(config): + path = Path(config["forbidden_path"]).resolve(strict=True) + verify_digest(path, config["forbidden_sha256"]) + values = tuple(json.loads(path.read_text(encoding="ascii"))) + if values != FORBIDDEN: + raise RuntimeError("forbidden loader list differs") + return values + + def validate_environment(environment, config): + names = sorted(environment) + if names != config["environment_allowlist"]: + raise RuntimeError(f"environment allowlist differs: {names}") + active = sorted(name for name in FORBIDDEN if environment.get(name)) + if active: + raise RuntimeError(f"forbidden loader environment is active: {active}") + return names + + def directory_receipts(workspace, build, direct_paths, uid): + workspace, build = Path(workspace), Path(build) + direct_paths = [Path(value) for value in direct_paths] + for path in (workspace, build, *direct_paths): + if not path.is_absolute() or path.resolve(strict=True) != path: + raise RuntimeError(f"directory is not canonical: {path}") + if build != workspace and workspace not in build.parents: + raise RuntimeError("build is outside workspace") + receipts = {} + for target in (build, *direct_paths): + if target != build and build not in target.parents: + raise RuntimeError(f"staging path is outside build: {target}") + current = workspace + candidates = [current] + for part in target.relative_to(workspace).parts: + current /= part + candidates.append(current) + for candidate in candidates: + if str(candidate) in receipts: + continue + info = candidate.lstat() + mode = stat.S_IMODE(info.st_mode) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + raise RuntimeError(f"staging ancestry is not a direct directory: {candidate}") + if info.st_uid != uid or mode & 0o022: + raise RuntimeError(f"staging ancestry owner or mode is invalid: {candidate}") + receipts[str(candidate)] = { + "device": info.st_dev, "gid": info.st_gid, "inode": info.st_ino, + "mode": mode, "uid": info.st_uid, + } + for path in direct_paths: + if receipts[str(path)]["mode"] != 0o700: + raise RuntimeError(f"staging directory mode is not 0700: {path}") + return receipts + + def load_config(): + path = Path(os.environ["DSV41_CI_CONFIG"]).resolve(strict=True) + verify_digest(path, os.environ["DSV41_CI_CONFIG_SHA256"]) + return json.loads(path.read_text(encoding="ascii")) + + def launch(): + config = load_config() + load_forbidden(config) + environment_names = validate_environment(os.environ, config) + state = current_state() + validate_state(state, config["runner_uid"], config["runner_gid"], config["runner_cap_bnd"]) + interpreter = validate_interpreter(config) + paths = config["paths"] + direct = [paths[name] for name in ("evidence", "hook", "log", "marker", "tmp")] + receipts = directory_receipts(paths["workspace"], paths["build"], direct, config["runner_uid"]) + ctest = str(Path(config["ctest"]).resolve(strict=True)) + if not os.access(ctest, os.X_OK): + raise RuntimeError("CTest is not executable") + preflight = { + "ctest": ctest, "environment_names": environment_names, + "format": "dsv41-ci-runner-preflight", "interpreter": interpreter, + "pid": os.getpid(), "ppid": os.getppid(), "secure_directories": receipts, + "state": state, "timestamp_monotonic_ns": time.monotonic_ns(), "version": 1, + } + path = Path(paths["evidence"]) / "runner-preflight.json" + digest = write_exclusive(path, preflight) + path.chmod(0o444) + environment = dict(os.environ) + environment["DSV41_CI_PREFLIGHT"] = str(path) + environment["DSV41_CI_PREFLIGHT_SHA256"] = digest + os.execve(ctest, [ + ctest, "--test-dir", paths["build"], "-R", "^test-deepseek41-trace$", + "--output-on-failure", "--verbose", "--no-tests=error", + "--output-junit", str(Path(paths["evidence"]) / "test-deepseek41-trace-ctest.xml"), + ], environment) + + def rejected(function): + try: + function() + except Exception: + return + raise RuntimeError("negative fixture was accepted") + + def self_test(output): + parsed = read_status( + "Uid:\t1001 1001 1001 1001\n" + "Gid:\t1001 1001 1001 1001\n" + "Groups:\t\n" + "CapInh:\t0000000000000000\n" + "CapPrm:\t0000000000000000\n" + "CapEff:\t0000000000000000\n" + "CapBnd:\t000000000000000f\n" + "CapAmb:\t0000000000000000\n" + "NoNewPrivs:\t1\n") + expected_parsed = { + "cap_amb": 0, "cap_bnd": 15, "cap_eff": 0, "cap_inh": 0, "cap_prm": 0, + "gid": [1001] * 4, "groups": [], "no_new_privs": 1, "uid": [1001] * 4, + } + if parsed != expected_parsed: + raise RuntimeError("accepted /proc/self/status fixture differs") + valid = { + "cap_amb": 0, "cap_bnd": 15, "cap_eff": 0, "cap_inh": 0, "cap_prm": 0, + "gid": [1001] * 4, "groups": [], "no_new_privs": 1, + "os_egid": 1001, "os_euid": 1001, "os_gid": 1001, + "os_groups": [], "os_uid": 1001, "uid": [1001] * 4, + } + validate_state(valid, 1001, 1001, 15) + for mutation in ( + {"groups": [5]}, {"os_groups": [5]}, + {"uid": [1002] * 4}, {"gid": [1002] * 4}, + {"cap_inh": 1}, {"cap_prm": 1}, {"cap_eff": 1}, {"cap_amb": 1}, + {"cap_bnd": 14}, {"no_new_privs": 0}): + changed = dict(valid) + changed.update(mutation) + rejected(lambda changed=changed: validate_state(changed, 1001, 1001, 15)) + validate_interpreter_values("/python3.11", "/python3.11", "/python3.11", "3.11", "3.11") + rejected(lambda: validate_interpreter_values("python3.11", "/python3.11", "/python3.11", "3.11", "3.11")) + rejected(lambda: validate_interpreter_values("/python3.11", "/other", "/python3.11", "3.11", "3.11")) + rejected(lambda: validate_interpreter_values("/python3.11", "/python3.11", "/python3.11", "3.12", "3.11")) + config = {"environment_allowlist": ["HOME"]} + validate_environment({"HOME": "/home/runner"}, config) + rejected(lambda: validate_environment({"HOME": "/home/runner", "LD_LIBRARY_PATH": "/lib"}, config)) + rejected(lambda: validate_environment( + {"HOME": "/home/runner", "LD_LIBRARY_PATH": "/lib"}, + {"environment_allowlist": ["HOME", "LD_LIBRARY_PATH"]})) + root = Path(output).resolve().parent / "gate-self-test" + root.mkdir(mode=0o700) + with tempfile.TemporaryDirectory(dir=root) as temp: + workspace = Path(temp) / "workspace" + build = workspace / "build" + direct = build / "direct" + direct.mkdir(parents=True, mode=0o700) + workspace.chmod(0o700) + build.chmod(0o700) + directory_receipts(workspace, build, [direct], os.getuid()) + direct.chmod(0o770) + rejected(lambda: directory_receipts(workspace, build, [direct], os.getuid())) + direct.chmod(0o702) + rejected(lambda: directory_receipts(workspace, build, [direct], os.getuid())) + direct.chmod(0o700) + link = build / "link" + link.symlink_to(direct, target_is_directory=True) + rejected(lambda: directory_receipts(workspace, build, [link], os.getuid())) + marker = build / "exclusive.json" + write_exclusive(marker, {"value": 1}) + rejected(lambda: write_exclusive(marker, {"value": 2})) + digest = hashlib.sha256(marker.read_bytes()).hexdigest() + marker.chmod(0o600) + marker.write_text('{"value":2}\n', encoding="ascii") + rejected(lambda: verify_digest(marker, digest)) + Path(output).write_text(json.dumps({ + "environment": "PASS", "exclusive_and_tamper": "PASS", + "interpreter": "PASS", "secure_ancestry": "PASS", + "state_matrix": "PASS", + }, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + + if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--self-test": + self_test(sys.argv[2]) + elif sys.argv[1:] == ["--launch"]: + launch() + else: + raise SystemExit("invalid fixture gate arguments") + PY + cat > "$ci_root/hook/sitecustomize.py" <<'PY' + import hashlib + import importlib.util + import json + import os + import sys + import time + import unittest + from pathlib import Path + + def load_gate(): + path = Path(os.environ["DSV41_CI_GATE"]).resolve(strict=True) + if hashlib.sha256(path.read_bytes()).hexdigest() != os.environ["DSV41_CI_GATE_SHA256"]: + raise RuntimeError("fixture gate SHA-256 mismatch") + spec = importlib.util.spec_from_file_location("dsv41_ci_fixture_gate", path) + if spec is None or spec.loader is None: + raise RuntimeError("cannot load fixture gate") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + GATE = load_gate() + CONFIG = GATE.load_config() + HOOK_PATH = Path(__file__).resolve(strict=True) + HOOK_SHA256 = GATE.verify_digest(HOOK_PATH, os.environ["DSV41_CI_HOOK_SHA256"]) + SOURCE = Path(CONFIG["source"]).resolve(strict=True) + MARKER_DIR = Path(CONFIG["paths"]["marker"]).resolve(strict=True) + TEST_CLASS = "TraceFormatTests" + TEST_METHOD = "test_native_watchdog_validation_crosses_private_pid_namespace" + TEST_ID = f"__main__.{TEST_CLASS}.{TEST_METHOD}" + BINDINGS = ( + "DSV41_NATIVE_TRACE_BINARY", "DSV41_NATIVE_CONTAINMENT_HELPER", + "DSV41_NATIVE_MANIFEST_BINARY", "DSV41_NATIVE_INJECT_LIBRARY", + ) + ORIGINAL_CALL = unittest.TestCase._callTestMethod + ORIGINAL_SKIP = unittest.TextTestResult.addSkip + + def preflight(): + path = Path(os.environ["DSV41_CI_PREFLIGHT"]).resolve(strict=True) + digest = GATE.verify_digest(path, os.environ["DSV41_CI_PREFLIGHT_SHA256"]) + return path, digest, json.loads(path.read_text(encoding="ascii")) - def _call_test_method(self, method): + def live_evidence(): + path, digest, before = preflight() + state = GATE.current_state() + GATE.validate_state(state, CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) + interpreter = GATE.validate_interpreter(CONFIG) + if any(os.environ.get(name) for name in GATE.FORBIDDEN): + raise RuntimeError("target process has a forbidden loader environment") + paths = CONFIG["paths"] + direct = [paths[name] for name in ("evidence", "hook", "log", "marker", "tmp")] + receipts = GATE.directory_receipts(paths["workspace"], paths["build"], direct, CONFIG["runner_uid"]) + if state != before["state"] or interpreter != before["interpreter"] or receipts != before["secure_directories"]: + raise RuntimeError("target process differs from preflight") + return { + "active_forbidden_loader_environment": [], + "interpreter": interpreter, + "preflight_path": str(path), + "preflight_sha256": digest, + "secure_directories": receipts, + "state": state, + } + + def call_test_method(self, method): function = getattr(method, "__func__", method) code = getattr(function, "__code__", None) source = Path(code.co_filename).resolve(strict=True) if code is not None else None - if self.__class__.__name__ == _TEST_CLASS and method.__name__ == _TEST_METHOD and source == _SOURCE: + if self.__class__.__name__ == TEST_CLASS and method.__name__ == TEST_METHOD and source == SOURCE: bindings = {} - for name in _BINDINGS: + for name in BINDINGS: value = os.environ.get(name) if not value: raise RuntimeError(f"missing {name} binding") bindings[name] = str(Path(value).resolve(strict=True)) payload = { - "bindings": bindings, - "event": "test_method_entry", - "first_line": code.co_firstlineno, - "function": _TEST_METHOD, - "pid": os.getpid(), - "ppid": os.getppid(), - "source": str(source), + **live_evidence(), "bindings": bindings, "event": "test_method_entry", + "first_line": code.co_firstlineno, "function": TEST_METHOD, + "gate_sha256": os.environ["DSV41_CI_GATE_SHA256"], + "hook_sha256": HOOK_SHA256, + "pid": os.getpid(), "ppid": os.getppid(), "source": str(source), "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), - "test_id": self.id(), - "timestamp_monotonic_ns": time.monotonic_ns(), + "test_id": self.id(), "timestamp_monotonic_ns": time.monotonic_ns(), } try: - _write_marker(_MARKER_DIR / "entry.json", payload) + GATE.write_exclusive(MARKER_DIR / "entry.json", payload) except FileExistsError: - duplicate = _MARKER_DIR / f"duplicate-{os.getpid()}-{time.monotonic_ns()}.json" - _write_marker(duplicate, payload) - raise RuntimeError(f"{_TEST_ID} entered more than once") - return _ORIGINAL_CALL_TEST_METHOD(self, method) + GATE.write_exclusive( + MARKER_DIR / f"duplicate-{os.getpid()}-{time.monotonic_ns()}.json", payload) + raise RuntimeError(f"{TEST_ID} entered more than once") + return ORIGINAL_CALL(self, method) - def _add_skip(self, test, reason): + def add_skip(self, test, reason): method = getattr(test, getattr(test, "_testMethodName", ""), None) function = getattr(method, "__func__", method) code = getattr(function, "__code__", None) source = Path(code.co_filename).resolve(strict=True) if code is not None else None test_id = test.id() + _path, digest, _before = preflight() payload = { - "event": "test_skip", - "reason": reason, + "event": "test_skip", "preflight_sha256": digest, "reason": reason, "source": str(source) if source is not None else None, "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest() if source is not None else None, "test_id": test_id, } - digest = hashlib.sha256(test_id.encode("ascii")).hexdigest() - _write_marker(_MARKER_DIR / f"skip-{digest}.json", payload) - return _ORIGINAL_ADD_SKIP(self, test, reason) + GATE.write_exclusive( + MARKER_DIR / f"skip-{hashlib.sha256(test_id.encode('ascii')).hexdigest()}.json", payload) + return ORIGINAL_SKIP(self, test, reason) - unittest.TestCase._callTestMethod = _call_test_method - unittest.TextTestResult.addSkip = _add_skip + unittest.TestCase._callTestMethod = call_test_method + unittest.TextTestResult.addSkip = add_skip PY - chmod 0444 build/deepseek-v41-trace-hook/sitecustomize.py - cmake -E sha256sum build/deepseek-v41-trace-hook/sitecustomize.py > build/deepseek-v41-trace-hook/sitecustomize.sha256 - : > build/test-deepseek41-trace-ctest.log - PYTHONPATH="$PWD/build/deepseek-v41-trace-hook${PYTHONPATH:+:$PYTHONPATH}" \ - DSV41_CI_MARKER_DIR="$PWD/build/deepseek-v41-trace-marker" \ - DSV41_CI_TEST_SOURCE="$PWD/tests/test-deepseek41-trace.py" \ - ctest --test-dir build \ - -R '^test-deepseek41-trace$' \ - --output-on-failure \ - --verbose \ - --no-tests=error \ - --output-junit test-deepseek41-trace-ctest.xml \ - 2>&1 | tee build/test-deepseek41-trace-ctest.log - python3 - <<'PY' + cat > "$ci_root/evidence/postgate.py" <<'PY' + import ast import hashlib + import importlib.util import json + import os + import re import sys + import tempfile from pathlib import Path from xml.etree import ElementTree - report = ElementTree.parse("build/test-deepseek41-trace-ctest.xml").getroot() - expected = { - "tests": 1, - "failures": 0, - "disabled": 0, - "skipped": 0, - } - actual = { - name: int(report.attrib.get(name, -1)) - for name in expected - } - if actual != expected: - raise SystemExit(f"test-deepseek41-trace JUnit result is invalid: {actual}") - log = Path("build/test-deepseek41-trace-ctest.log").read_text(errors="replace") - if "***Skipped" in log or "***Not Run" in log: - raise SystemExit("test-deepseek41-trace was skipped or not run") - - metadata = json.loads(Path("build/test-deepseek41-trace-ctest-metadata.json").read_text()) - test = metadata["tests"][0] - properties = { - item["name"]: item["value"] - for item in test.get("properties", []) - } - environment = properties["ENVIRONMENT"] - if isinstance(environment, str): - environment = environment.split(";") - expected_bindings = {} - for value in environment: - name, setting = value.split("=", 1) - if name.startswith("DSV41_NATIVE_"): - expected_bindings[name] = str(Path(setting).resolve(strict=True)) - if set(expected_bindings) != { - "DSV41_NATIVE_TRACE_BINARY", - "DSV41_NATIVE_CONTAINMENT_HELPER", - "DSV41_NATIVE_MANIFEST_BINARY", - "DSV41_NATIVE_INJECT_LIBRARY", - }: - raise SystemExit("test-deepseek41-trace native bindings are invalid") - test_name = "TraceFormatTests.test_native_watchdog_validation_crosses_private_pid_namespace" - marker_dir = Path("build/deepseek-v41-trace-marker") - entries = list(marker_dir.glob("entry.json")) - duplicates = list(marker_dir.glob("duplicate-*.json")) - skips = list(marker_dir.glob("skip-*.json")) - unexpected = [ - path - for path in marker_dir.iterdir() - if path not in entries and path not in duplicates and path not in skips - ] - if len(entries) != 1 or duplicates or unexpected: - raise SystemExit( - f"private PID namespace/pidfd entry count is invalid: " - f"entries={len(entries)} duplicates={len(duplicates)} unexpected={len(unexpected)}") - marker = json.loads(entries[0].read_text()) - expected_source = Path("tests/test-deepseek41-trace.py").resolve(strict=True) - expected_marker = { - "bindings": expected_bindings, - "event": "test_method_entry", - "function": test_name.split(".", 1)[1], - "source": str(expected_source), - "source_sha256": hashlib.sha256(expected_source.read_bytes()).hexdigest(), - "test_id": f"__main__.{test_name}", - } - for name, value in expected_marker.items(): - if marker.get(name) != value: - raise SystemExit(f"private PID namespace/pidfd marker {name} is invalid") - if not isinstance(marker.get("first_line"), int) or marker["first_line"] <= 0: - raise SystemExit("private PID namespace/pidfd marker first_line is invalid") - if not isinstance(marker.get("pid"), int) or marker["pid"] <= 1: - raise SystemExit("private PID namespace/pidfd marker pid is invalid") - if not isinstance(marker.get("ppid"), int) or marker["ppid"] <= 0: - raise SystemExit("private PID namespace/pidfd marker ppid is invalid") - if not isinstance(marker.get("timestamp_monotonic_ns"), int) or marker["timestamp_monotonic_ns"] <= 0: - raise SystemExit("private PID namespace/pidfd marker timestamp is invalid") - if sys.platform != "linux": - raise SystemExit(f"test-deepseek41-trace requires Linux, got {sys.platform}") - if len(skips) != 1: - raise SystemExit(f"test-deepseek41-trace approved skip count is invalid: {len(skips)}") - approved_skip = json.loads(skips[0].read_text()) - expected_skip = { - "event": "test_skip", - "reason": "Linux uses subreaper and pidfd containment", - "source": str(expected_source), - "source_sha256": hashlib.sha256(expected_source.read_bytes()).hexdigest(), - "test_id": "__main__.TraceFormatTests.test_unproven_posix_containment_fails_closed_before_setsid_escape", + def load_gate(): + path = Path(os.environ["DSV41_CI_GATE"]).resolve(strict=True) + if hashlib.sha256(path.read_bytes()).hexdigest() != os.environ["DSV41_CI_GATE_SHA256"]: + raise RuntimeError("fixture gate SHA-256 mismatch") + spec = importlib.util.spec_from_file_location("dsv41_ci_fixture_gate_post", path) + if spec is None or spec.loader is None: + raise RuntimeError("cannot load fixture gate") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + GATE = load_gate() + CONFIG = GATE.load_config() + TARGET_METHOD = "test_native_watchdog_validation_crosses_private_pid_namespace" + TARGET_ID = f"__main__.TraceFormatTests.{TARGET_METHOD}" + APPROVED_SKIP_ID = "__main__.TraceFormatTests.test_unproven_posix_containment_fails_closed_before_setsid_escape" + BINDING_NAMES = { + "DSV41_NATIVE_TRACE_BINARY", "DSV41_NATIVE_CONTAINMENT_HELPER", + "DSV41_NATIVE_MANIFEST_BINARY", "DSV41_NATIVE_INJECT_LIBRARY", } - if approved_skip != expected_skip: - raise SystemExit("test-deepseek41-trace skip identity is invalid") + + def junit_values(path): + root = ElementTree.parse(path).getroot() + return {name: int(root.attrib.get(name, -1)) for name in ("tests", "failures", "disabled", "skipped")} + + def validate_outer(status, junit, log): + if status != 0: + raise RuntimeError(f"CTest failed with status {status}") + if junit != {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}: + raise RuntimeError(f"CTest JUnit result is invalid: {junit}") + if "***Skipped" in log or "***Not Run" in log: + raise RuntimeError("CTest was skipped or not run") + start_pattern = re.compile(r"^\s*Start\s+\d+:\s+test-deepseek41-trace\s*$") + starts = sum(start_pattern.fullmatch(line) is not None for line in log.splitlines()) + if starts != 1: + raise RuntimeError(f"registered CTest execution count is invalid: {starts}") + + def expected_bindings(metadata): + tests = metadata.get("tests", []) + if len(tests) != 1 or tests[0].get("name") != "test-deepseek41-trace": + raise RuntimeError("CTest metadata selector is invalid") + properties = {item["name"]: item["value"] for item in tests[0].get("properties", [])} + environment = properties.get("ENVIRONMENT", []) + if isinstance(environment, str): + environment = environment.split(";") + result = {} + for value in environment: + name, setting = value.split("=", 1) + if name.startswith("DSV41_NATIVE_"): + result[name] = str(Path(setting).resolve(strict=True)) + if set(result) != BINDING_NAMES: + raise RuntimeError("native bindings are invalid") + return result + + def target_first_line(source): + tree = ast.parse(source.read_text(encoding="ascii")) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "TraceFormatTests": + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)) and member.name == TARGET_METHOD: + return member.lineno + raise RuntimeError("target method is absent") + + def validate_counts(entries, duplicates, target_skips, total_skips): + if entries != 1 or duplicates != 0 or target_skips != 0 or total_skips != 1: + raise RuntimeError( + f"marker counts are invalid: entry={entries} duplicate={duplicates} " + f"target_skip={target_skips} skips={total_skips}") + + def validate_marker(status, paths): + evidence = Path(paths["evidence"]) + marker_dir = Path(paths["marker"]) + log = (Path(paths["log"]) / "test-deepseek41-trace-ctest.log").read_text(errors="replace") + validate_outer( + status, + junit_values(evidence / "test-deepseek41-trace-ctest.xml"), + log, + ) + metadata = json.loads((evidence / "test-deepseek41-trace-ctest-metadata.json").read_text()) + bindings = expected_bindings(metadata) + preflight_path = evidence / "runner-preflight.json" + preflight_sha = hashlib.sha256(preflight_path.read_bytes()).hexdigest() + preflight = json.loads(preflight_path.read_text(encoding="ascii")) + GATE.validate_state( + preflight["state"], CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) + if preflight["environment_names"] != CONFIG["environment_allowlist"]: + raise RuntimeError("preflight environment allowlist differs") + if preflight["ctest"] != str(Path(CONFIG["ctest"]).resolve(strict=True)): + raise RuntimeError("preflight CTest identity differs") + if preflight["interpreter"] != GATE.validate_interpreter(CONFIG): + raise RuntimeError("preflight Python identity differs") + direct = [paths[name] for name in ("evidence", "hook", "log", "marker", "tmp")] + receipts = GATE.directory_receipts( + paths["workspace"], paths["build"], direct, CONFIG["runner_uid"]) + if preflight["secure_directories"] != receipts: + raise RuntimeError("preflight secure directory evidence differs") + entries = list(marker_dir.glob("entry.json")) + duplicates = list(marker_dir.glob("duplicate-*.json")) + skips = list(marker_dir.glob("skip-*.json")) + unexpected = [ + path for path in marker_dir.iterdir() + if path not in entries and path not in duplicates and path not in skips + ] + skip_payloads = [json.loads(path.read_text(encoding="ascii")) for path in skips] + target_skips = sum(payload.get("test_id") == TARGET_ID for payload in skip_payloads) + validate_counts(len(entries), len(duplicates), target_skips, len(skips)) + if unexpected: + raise RuntimeError(f"unexpected marker artifacts: {unexpected}") + marker = json.loads(entries[0].read_text(encoding="ascii")) + source = Path(CONFIG["source"]).resolve(strict=True) + expected = { + "active_forbidden_loader_environment": [], + "bindings": bindings, + "event": "test_method_entry", + "first_line": target_first_line(source), + "function": TARGET_METHOD, + "gate_sha256": os.environ["DSV41_CI_GATE_SHA256"], + "hook_sha256": os.environ["DSV41_CI_HOOK_SHA256"], + "interpreter": preflight["interpreter"], + "preflight_path": str(preflight_path), + "preflight_sha256": preflight_sha, + "secure_directories": receipts, + "source": str(source), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "state": preflight["state"], + "test_id": TARGET_ID, + } + for name, value in expected.items(): + if marker.get(name) != value: + raise RuntimeError(f"target entry marker {name} differs") + if not isinstance(marker.get("pid"), int) or marker["pid"] <= 1: + raise RuntimeError("target entry marker PID is invalid") + if not isinstance(marker.get("ppid"), int) or marker["ppid"] <= 0: + raise RuntimeError("target entry marker PPID is invalid") + if not isinstance(marker.get("timestamp_monotonic_ns"), int) or marker["timestamp_monotonic_ns"] <= 0: + raise RuntimeError("target entry marker timestamp is invalid") + approved = { + "event": "test_skip", + "preflight_sha256": preflight_sha, + "reason": "Linux uses subreaper and pidfd containment", + "source": str(source), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "test_id": APPROVED_SKIP_ID, + } + if skip_payloads != [approved]: + raise RuntimeError("approved skip identity differs") + return { + "ctest_execution_count": 1, + "duplicate_entry_count": 0, + "forbidden_loader_environment": [], + "junit": {"disabled": 0, "failures": 0, "skipped": 0, "tests": 1}, + "preflight_sha256": preflight_sha, + "result": "PASS", + "target_entry_count": 1, + "target_skip_count": 0, + } + + def rejected(function): + try: + function() + except Exception: + return + raise RuntimeError("negative postgate fixture was accepted") + + def self_test(output): + validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n") + rejected(lambda: validate_outer( + 1, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 1, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 1}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 1, "skipped": 0}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n***Not Run\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, "")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + "Start 35: test-deepseek41-trace\nStart 35: test-deepseek41-trace\n")) + validate_counts(1, 0, 0, 1) + for counts in ((0, 0, 0, 1), (1, 1, 0, 1), (1, 0, 1, 1), (1, 0, 0, 2)): + rejected(lambda counts=counts: validate_counts(*counts)) + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp) / "marker.json" + GATE.write_exclusive(marker, {"preflight_sha256": "a"}) + rejected(lambda: GATE.write_exclusive(marker, {"preflight_sha256": "b"})) + rejected(lambda: GATE.verify_digest(marker, "0" * 64)) + Path(output).write_text(json.dumps({ + "duplicate_and_race": "PASS", "junit_failure_skip_notrun": "PASS", + "preflight_spoof": "PASS", "target_nonentry_skip_duplicate": "PASS", + }, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + + def validate(status, output): + try: + receipt = validate_marker(status, CONFIG["paths"]) + except Exception as error: + receipt = {"error": str(error), "result": "FAIL"} + GATE.write_exclusive(Path(output), receipt) + raise + GATE.write_exclusive(Path(output), receipt) + + if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--self-test": + self_test(sys.argv[2]) + elif len(sys.argv) == 4 and sys.argv[1] == "--validate": + validate(int(sys.argv[2]), sys.argv[3]) + else: + raise SystemExit("invalid postgate arguments") PY + chmod 0444 \ + "$ci_root/evidence/fixture-config.json" \ + "$ci_root/evidence/fixture_gate.py" \ + "$ci_root/evidence/forbidden-loader-environment.json" \ + "$ci_root/evidence/postgate.py" \ + "$ci_root/hook/sitecustomize.py" + export DSV41_CI_CONFIG="$ci_root/evidence/fixture-config.json" + export DSV41_CI_CONFIG_SHA256="$(sha256sum "$DSV41_CI_CONFIG" | cut -d' ' -f1)" + export DSV41_CI_GATE="$ci_root/evidence/fixture_gate.py" + export DSV41_CI_GATE_SHA256="$(sha256sum "$DSV41_CI_GATE" | cut -d' ' -f1)" + export DSV41_CI_HOOK_SHA256="$(sha256sum "$ci_root/hook/sitecustomize.py" | cut -d' ' -f1)" + printf '%s %s\n' "$DSV41_CI_GATE_SHA256" "$DSV41_CI_GATE" > "$ci_root/evidence/fixture_gate.sha256" + printf '%s %s\n' "$DSV41_CI_HOOK_SHA256" "$ci_root/hook/sitecustomize.py" > "$ci_root/evidence/sitecustomize.sha256" + "$selected_interpreter" "$ci_root/evidence/fixture_gate.py" \ + --self-test "$ci_root/evidence/fixture-gate-self-test.json" + "$selected_interpreter" "$ci_root/evidence/postgate.py" \ + --self-test "$ci_root/evidence/postgate-self-test.json" + : > "$ci_root/log/test-deepseek41-trace-ctest.log" + set +e + sudo -n -- /usr/bin/setpriv \ + --reuid="$runner_uid" \ + --regid="$runner_gid" \ + --clear-groups \ + --inh-caps=-all \ + --ambient-caps=-all \ + --no-new-privs \ + -- /usr/bin/env -i \ + "PATH=${selected_interpreter%/*}:/usr/bin:/bin" \ + "HOME=$HOME" \ + "LANG=C" \ + "LC_ALL=C" \ + "TMPDIR=$ci_root/tmp" \ + "PYTHONDONTWRITEBYTECODE=1" \ + "PYTHONCOERCECLOCALE=0" \ + "PYTHONPATH=$ci_root/hook" \ + "PYTHONUTF8=1" \ + "DSV41_CI_CONFIG=$DSV41_CI_CONFIG" \ + "DSV41_CI_CONFIG_SHA256=$DSV41_CI_CONFIG_SHA256" \ + "DSV41_CI_GATE=$DSV41_CI_GATE" \ + "DSV41_CI_GATE_SHA256=$DSV41_CI_GATE_SHA256" \ + "DSV41_CI_HOOK_SHA256=$DSV41_CI_HOOK_SHA256" \ + "$selected_interpreter" "$ci_root/evidence/fixture_gate.py" --launch \ + 2>&1 | tee "$ci_root/log/test-deepseek41-trace-ctest.log" + ctest_status="${PIPESTATUS[0]}" + set -e + "$selected_interpreter" "$ci_root/evidence/postgate.py" \ + --validate "$ctest_status" "$ci_root/evidence/postgate-result.json" - name: Upload DeepSeek V4.1 trace CTest evidence if: ${{ always() }} @@ -387,13 +968,7 @@ jobs: with: name: deepseek-v41-trace-ctest-${{ github.run_id }}-${{ github.run_attempt }} path: | - build/test-deepseek41-trace-ctest-metadata.json - build/test-deepseek41-trace-python.json - build/test-deepseek41-trace-ctest.log - build/test-deepseek41-trace-ctest.xml - build/deepseek-v41-trace-hook/sitecustomize.py - build/deepseek-v41-trace-hook/sitecustomize.sha256 - build/deepseek-v41-trace-marker/ + build/deepseek-v41-trace-ci/ if-no-files-found: error retention-days: 7 From d1914596183accc84e7a684a9280161aff8e88ff Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 10:04:37 -0700 Subject: [PATCH 55/56] ci : validate CTest trace numbering Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/server.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 79ce24d52a52..977ac2a865aa 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -728,7 +728,7 @@ jobs: raise RuntimeError(f"CTest JUnit result is invalid: {junit}") if "***Skipped" in log or "***Not Run" in log: raise RuntimeError("CTest was skipped or not run") - start_pattern = re.compile(r"^\s*Start\s+\d+:\s+test-deepseek41-trace\s*$") + start_pattern = re.compile(r"^\s*Start\s+[1-9]\d*:\s+test-deepseek41-trace\s*$") starts = sum(start_pattern.fullmatch(line) is not None for line in log.splitlines()) if starts != 1: raise RuntimeError(f"registered CTest execution count is invalid: {starts}") @@ -756,9 +756,13 @@ jobs: if isinstance(node, ast.ClassDef) and node.name == "TraceFormatTests": for member in node.body: if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)) and member.name == TARGET_METHOD: - return member.lineno + return min([member.lineno, *(decorator.lineno for decorator in member.decorator_list)]) raise RuntimeError("target method is absent") + def validate_first_line(actual, expected): + if actual != expected: + raise RuntimeError(f"target first line differs: actual={actual} expected={expected}") + def validate_counts(entries, duplicates, target_skips, total_skips): if entries != 1 or duplicates != 0 or target_skips != 0 or total_skips != 1: raise RuntimeError( @@ -810,7 +814,6 @@ jobs: "active_forbidden_loader_environment": [], "bindings": bindings, "event": "test_method_entry", - "first_line": target_first_line(source), "function": TARGET_METHOD, "gate_sha256": os.environ["DSV41_CI_GATE_SHA256"], "hook_sha256": os.environ["DSV41_CI_HOOK_SHA256"], @@ -826,6 +829,7 @@ jobs: for name, value in expected.items(): if marker.get(name) != value: raise RuntimeError(f"target entry marker {name} differs") + validate_first_line(marker.get("first_line"), target_first_line(source)) if not isinstance(marker.get("pid"), int) or marker["pid"] <= 1: raise RuntimeError("target entry marker PID is invalid") if not isinstance(marker.get("ppid"), int) or marker["ppid"] <= 0: @@ -884,6 +888,21 @@ jobs: rejected(lambda: validate_outer( 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, "Start 35: test-deepseek41-trace\nStart 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + "Start 0: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + "Start 35: wrong-test\n")) + decorated = ast.parse( + "class TraceFormatTests:\n" + " @staticmethod\n" + " def test_native_watchdog_validation_crosses_private_pid_namespace():\n" + " pass\n") + member = decorated.body[0].body[0] + first_line = min([member.lineno, *(item.lineno for item in member.decorator_list)]) + validate_first_line(first_line, 2) + rejected(lambda: validate_first_line(member.lineno, 2)) validate_counts(1, 0, 0, 1) for counts in ((0, 0, 0, 1), (1, 1, 0, 1), (1, 0, 1, 1), (1, 0, 0, 2)): rejected(lambda counts=counts: validate_counts(*counts)) From 0ab98b26bc804cacd1c62669fc72de735affadcf Mon Sep 17 00:00:00 2001 From: Jerome Coste Date: Mon, 14 Sep 2026 11:38:30 -0700 Subject: [PATCH 56/56] ci : capture hosted namespace policy controls Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1 Assisted-by: GitHub Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/server.yml | 574 ++++++++++++++++++++++++++++++++++- 1 file changed, 567 insertions(+), 7 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 977ac2a865aa..f385499a9690 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -309,6 +309,22 @@ jobs: "DYLD_VERSIONED_LIBRARY_PATH", "GGML_BACKEND_PATH", "LD_AUDIT", "LD_LIBRARY_PATH", "LD_PRELOAD", ) + HOST_POLICY_STATUS_FIELDS = ( + "Uid", "Gid", "Groups", "CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb", + "NoNewPrivs", "Seccomp", "Seccomp_filters", + ) + HOST_POLICY_CONTROL_PATHS = ( + "/proc/self/attr/current", + "/sys/module/apparmor/parameters/enabled", + "/proc/sys/kernel/unprivileged_userns_clone", + "/proc/sys/user/max_user_namespaces", + "/proc/sys/kernel/apparmor_restrict_unprivileged_userns", + "/sys/kernel/security/lsm", + "/sys/kernel/security/apparmor/profiles", + "/sys/kernel/security", + "/sys/kernel/security/apparmor", + "/sys/kernel/security/apparmor/features", + ) def bytes_for(value): return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") @@ -362,6 +378,354 @@ jobs: }) return state + def error_snapshot(path, operation, error, metadata=None): + result = { + "error": str(error), + "error_type": type(error).__name__, + "errno": getattr(error, "errno", None), + "operation": operation, + "path": str(path), + "status": "unreadable", + } + if metadata is not None: + result.update(metadata) + return result + + def path_metadata(path, info): + kind = "directory" if stat.S_ISDIR(info.st_mode) else "file" if stat.S_ISREG(info.st_mode) else "other" + return { + "gid": info.st_gid, + "kind": kind, + "mode": stat.S_IMODE(info.st_mode), + "uid": info.st_uid, + } + + def snapshot_path(value): + path = Path(value) + try: + info = path.lstat() + except FileNotFoundError: + return {"path": str(path), "status": "absent"} + except OSError as error: + return error_snapshot(path, "lstat", error) + metadata = path_metadata(path, info) + if metadata["kind"] == "directory": + try: + entries = sorted(os.listdir(path)) + except OSError as error: + return error_snapshot(path, "listdir", error, metadata) + encoded = bytes_for(entries) + return { + **metadata, + "entries": entries, + "entries_sha256": hashlib.sha256(encoded).hexdigest(), + "path": str(path), + "status": "present_readable", + } + if metadata["kind"] != "file": + return { + **metadata, + "path": str(path), + "status": "present_readable", + } + try: + data = path.read_bytes() + except OSError as error: + return error_snapshot(path, "read", error, metadata) + try: + content = data.decode("ascii") + encoding = "ascii" + except UnicodeDecodeError: + content = data.hex() + encoding = "hex" + return { + **metadata, + "bytes": len(data), + "content": content, + "content_encoding": encoding, + "content_sha256": hashlib.sha256(data).hexdigest(), + "path": str(path), + "status": "present_readable", + } + + def validate_snapshot(snapshot, expected_path): + if snapshot.get("path") != expected_path: + raise RuntimeError(f"host policy path differs: {snapshot.get('path')} != {expected_path}") + status = snapshot.get("status") + if status == "absent": + if set(snapshot) != {"path", "status"}: + raise RuntimeError(f"absent host policy control has extra fields: {expected_path}") + return + if status == "unreadable": + allowed = { + "error", "error_type", "errno", "operation", "path", "status", + "gid", "kind", "mode", "uid", + } + if not set(snapshot) <= allowed: + raise RuntimeError(f"unreadable host policy control has extra fields: {expected_path}") + if not isinstance(snapshot.get("operation"), str) or not snapshot["operation"]: + raise RuntimeError(f"unreadable host policy control has no operation: {expected_path}") + if not isinstance(snapshot.get("error_type"), str) or not snapshot["error_type"]: + raise RuntimeError(f"unreadable host policy control has no error type: {expected_path}") + if not isinstance(snapshot.get("error"), str) or not snapshot["error"]: + raise RuntimeError(f"unreadable host policy control has no error: {expected_path}") + if snapshot.get("errno") is not None and not isinstance(snapshot["errno"], int): + raise RuntimeError(f"unreadable host policy control errno is invalid: {expected_path}") + metadata = [name in snapshot for name in ("mode", "uid", "gid", "kind")] + if any(metadata) and not all(metadata): + raise RuntimeError(f"unreadable host policy control metadata is incomplete: {expected_path}") + if all(metadata): + if snapshot["kind"] not in ("directory", "file", "other"): + raise RuntimeError(f"unreadable host policy control kind is invalid: {expected_path}") + for name in ("mode", "uid", "gid"): + if not isinstance(snapshot[name], int) or snapshot[name] < 0: + raise RuntimeError(f"unreadable host policy control {name} is invalid: {expected_path}") + return + if status != "present_readable": + raise RuntimeError(f"host policy control status is invalid: {expected_path}") + for name in ("mode", "uid", "gid"): + if not isinstance(snapshot.get(name), int) or snapshot[name] < 0: + raise RuntimeError(f"host policy control {name} is invalid: {expected_path}") + kind = snapshot.get("kind") + if kind == "directory": + expected = { + "entries", "entries_sha256", "gid", "kind", "mode", "path", "status", "uid", + } + if set(snapshot) != expected: + raise RuntimeError(f"host policy directory fields differ: {expected_path}") + entries = snapshot.get("entries") + if not isinstance(entries, list) or entries != sorted(entries) or not all( + isinstance(item, str) for item in entries): + raise RuntimeError(f"host policy directory entries are invalid: {expected_path}") + if snapshot.get("entries_sha256") != hashlib.sha256(bytes_for(entries)).hexdigest(): + raise RuntimeError(f"host policy directory entry hash differs: {expected_path}") + return + if kind == "other": + if set(snapshot) != {"gid", "kind", "mode", "path", "status", "uid"}: + raise RuntimeError(f"host policy other fields differ: {expected_path}") + return + if kind != "file": + raise RuntimeError(f"host policy control kind is invalid: {expected_path}") + expected = { + "bytes", "content", "content_encoding", "content_sha256", + "gid", "kind", "mode", "path", "status", "uid", + } + if set(snapshot) != expected: + raise RuntimeError(f"host policy file fields differ: {expected_path}") + encoding = snapshot.get("content_encoding") + content = snapshot.get("content") + if not isinstance(content, str) or encoding not in ("ascii", "hex"): + raise RuntimeError(f"host policy control content is invalid: {expected_path}") + try: + data = content.encode("ascii") if encoding == "ascii" else bytes.fromhex(content) + except (UnicodeEncodeError, ValueError) as error: + raise RuntimeError(f"host policy control encoding is invalid: {expected_path}") from error + if snapshot.get("bytes") != len(data): + raise RuntimeError(f"host policy control byte count differs: {expected_path}") + if snapshot.get("content_sha256") != hashlib.sha256(data).hexdigest(): + raise RuntimeError(f"host policy control hash differs: {expected_path}") + + def snapshot_ascii(snapshot): + validate_snapshot(snapshot, snapshot.get("path")) + if snapshot.get("status") != "present_readable" or snapshot.get("kind") != "file": + return None + if snapshot.get("content_encoding") != "ascii": + raise RuntimeError(f"host policy control is not ASCII: {snapshot['path']}") + return snapshot["content"] + + def status_field_evidence(status_snapshot): + text = snapshot_ascii(status_snapshot) + if text is None: + return { + name: {"status": "unavailable"} + for name in HOST_POLICY_STATUS_FIELDS + } + fields = {} + for line in text.splitlines(): + name, separator, value = line.partition(":") + if separator: + fields[name] = value.strip() + return { + name: {"raw": fields[name], "status": "present"} if name in fields else {"status": "absent"} + for name in HOST_POLICY_STATUS_FIELDS + } + + def root_mount_evidence(mountinfo_snapshot): + text = snapshot_ascii(mountinfo_snapshot) + if text is None: + return [] + result = [] + for line in text.splitlines(): + before, separator, after = line.partition(" - ") + fields = before.split() + suffix = after.split() + if not separator or len(fields) < 6 or len(suffix) < 3 or fields[4] != "/": + continue + result.append({ + "filesystem_type": suffix[0], + "major_minor": fields[2], + "mount_id": int(fields[0]), + "mount_options": fields[5], + "mount_point": fields[4], + "mount_source": suffix[1], + "optional_fields": fields[6:], + "parent_id": int(fields[1]), + "raw_line": line, + "root": fields[3], + "super_options": suffix[2:], + }) + return result + + def capture_host_policy_diagnostic(state): + try: + uname = os.uname() + uname_evidence = { + "machine": uname.machine, + "nodename": uname.nodename, + "release": uname.release, + "status": "available", + "sysname": uname.sysname, + "version": uname.version, + } + except OSError as error: + uname_evidence = { + "error": str(error), + "error_type": type(error).__name__, + "errno": error.errno, + "status": "unavailable", + } + status_snapshot = snapshot_path("/proc/self/status") + mountinfo_snapshot = snapshot_path("/proc/self/mountinfo") + diagnostic = { + "captured_state": state, + "controls": { + path: snapshot_path(path) + for path in HOST_POLICY_CONTROL_PATHS + }, + "format": "dsv41-host-policy-diagnostic", + "proc_self_status": { + "fields": status_field_evidence(status_snapshot), + "snapshot": status_snapshot, + }, + "root_mount_propagation": { + "root_mounts": root_mount_evidence(mountinfo_snapshot), + "snapshot": mountinfo_snapshot, + }, + "uname": uname_evidence, + "version": 1, + } + validate_host_policy_diagnostic(diagnostic, state) + return diagnostic + + def parse_decimal_field(fields, name): + item = fields[name] + if item.get("status") == "absent": + return None + if item.get("status") != "present" or not isinstance(item.get("raw"), str): + raise RuntimeError(f"host policy status field is invalid: {name}") + try: + return int(item["raw"], 10) + except ValueError as error: + raise RuntimeError(f"host policy status field is not decimal: {name}") from error + + def parse_vector_field(fields, name): + item = fields[name] + if item.get("status") != "present" or not isinstance(item.get("raw"), str): + raise RuntimeError(f"host policy status field is invalid: {name}") + try: + return [int(value, 10) for value in item["raw"].split()] + except ValueError as error: + raise RuntimeError(f"host policy status vector is invalid: {name}") from error + + def parse_hex_field(fields, name): + item = fields[name] + if item.get("status") != "present" or not isinstance(item.get("raw"), str): + raise RuntimeError(f"host policy status field is invalid: {name}") + try: + return int(item["raw"], 16) + except ValueError as error: + raise RuntimeError(f"host policy status field is not hexadecimal: {name}") from error + + def validate_host_policy_diagnostic(diagnostic, state): + if diagnostic.get("format") != "dsv41-host-policy-diagnostic" or diagnostic.get("version") != 1: + raise RuntimeError("host policy diagnostic schema differs") + if diagnostic.get("captured_state") != state: + raise RuntimeError("host policy diagnostic state differs") + uname = diagnostic.get("uname", {}) + if uname.get("status") == "available": + if not all(isinstance(uname.get(name), str) for name in ( + "sysname", "nodename", "release", "version", "machine")): + raise RuntimeError("host policy uname evidence is invalid") + elif uname.get("status") == "unavailable": + if not isinstance(uname.get("error_type"), str) or not isinstance(uname.get("error"), str): + raise RuntimeError("host policy uname error is invalid") + else: + raise RuntimeError("host policy uname status is invalid") + status = diagnostic.get("proc_self_status", {}) + snapshot = status.get("snapshot", {}) + validate_snapshot(snapshot, "/proc/self/status") + fields = status.get("fields") + if not isinstance(fields, dict) or set(fields) != set(HOST_POLICY_STATUS_FIELDS): + raise RuntimeError("host policy status fields differ") + if snapshot.get("status") != "present_readable": + if any(value != {"status": "unavailable"} for value in fields.values()): + raise RuntimeError("unavailable host policy status has values") + else: + if parse_vector_field(fields, "Uid") != state["uid"]: + raise RuntimeError("host policy Uid differs") + if parse_vector_field(fields, "Gid") != state["gid"]: + raise RuntimeError("host policy Gid differs") + if parse_vector_field(fields, "Groups") != state["groups"]: + raise RuntimeError("host policy Groups differs") + for field, name in ( + ("CapInh", "cap_inh"), ("CapPrm", "cap_prm"), ("CapEff", "cap_eff"), + ("CapBnd", "cap_bnd"), ("CapAmb", "cap_amb")): + if parse_hex_field(fields, field) != state[name]: + raise RuntimeError(f"host policy {field} differs") + if parse_decimal_field(fields, "NoNewPrivs") != state["no_new_privs"]: + raise RuntimeError("host policy NoNewPrivs differs") + for name in ("Seccomp", "Seccomp_filters"): + parse_decimal_field(fields, name) + controls = diagnostic.get("controls") + if not isinstance(controls, dict) or set(controls) != set(HOST_POLICY_CONTROL_PATHS): + raise RuntimeError("host policy control paths differ") + for path in HOST_POLICY_CONTROL_PATHS: + validate_snapshot(controls[path], path) + mount = diagnostic.get("root_mount_propagation", {}) + validate_snapshot(mount.get("snapshot", {}), "/proc/self/mountinfo") + roots = mount.get("root_mounts") + if not isinstance(roots, list): + raise RuntimeError("host policy root mounts are invalid") + if mount["snapshot"].get("status") == "present_readable" and not roots: + raise RuntimeError("host policy root mount is absent") + if mount["snapshot"].get("status") != "present_readable" and roots: + raise RuntimeError("unavailable host policy mount has root entries") + for item in roots: + if item.get("mount_point") != "/" or not isinstance(item.get("raw_line"), str): + raise RuntimeError("host policy root mount entry is invalid") + + def validate_host_policy_file(binding, state, expected_path): + path = Path(binding.get("path", "")) + if not path.is_absolute() or path.resolve(strict=True) != path or path != Path(expected_path): + raise RuntimeError("host policy diagnostic path is invalid") + info = path.lstat() + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + raise RuntimeError("host policy diagnostic is not a direct file") + if stat.S_IMODE(info.st_mode) != 0o444 or binding.get("mode") != 0o444: + raise RuntimeError("host policy diagnostic mode differs") + digest = verify_digest(path, binding.get("sha256")) + diagnostic = json.loads(path.read_text(encoding="ascii")) + validate_host_policy_diagnostic(diagnostic, state) + if binding.get("format") != diagnostic["format"] or binding.get("version") != diagnostic["version"]: + raise RuntimeError("host policy diagnostic binding schema differs") + return { + "format": diagnostic["format"], + "mode": 0o444, + "path": str(path), + "sha256": digest, + "version": diagnostic["version"], + } + def validate_state(state, uid, gid, cap_bnd): if uid <= 0 or gid <= 0: raise RuntimeError("runner identity is privileged") @@ -465,9 +829,21 @@ jobs: ctest = str(Path(config["ctest"]).resolve(strict=True)) if not os.access(ctest, os.X_OK): raise RuntimeError("CTest is not executable") + diagnostic_path = Path(paths["evidence"]) / "host-policy-diagnostic.json" + diagnostic = capture_host_policy_diagnostic(state) + diagnostic_sha256 = write_exclusive(diagnostic_path, diagnostic) + diagnostic_path.chmod(0o444) + diagnostic_binding = validate_host_policy_file({ + "format": diagnostic["format"], + "mode": 0o444, + "path": str(diagnostic_path), + "sha256": diagnostic_sha256, + "version": diagnostic["version"], + }, state, diagnostic_path) preflight = { "ctest": ctest, "environment_names": environment_names, "format": "dsv41-ci-runner-preflight", "interpreter": interpreter, + "host_policy_diagnostic": diagnostic_binding, "pid": os.getpid(), "ppid": os.getppid(), "secure_directories": receipts, "state": state, "timestamp_monotonic_ns": time.monotonic_ns(), "version": 1, } @@ -532,6 +908,169 @@ jobs: rejected(lambda: validate_environment( {"HOME": "/home/runner", "LD_LIBRARY_PATH": "/lib"}, {"environment_allowlist": ["HOME", "LD_LIBRARY_PATH"]})) + state_fields = { + "CapAmb": {"raw": "0000000000000000", "status": "present"}, + "CapBnd": {"raw": "000000000000000f", "status": "present"}, + "CapEff": {"raw": "0000000000000000", "status": "present"}, + "CapInh": {"raw": "0000000000000000", "status": "present"}, + "CapPrm": {"raw": "0000000000000000", "status": "present"}, + "Gid": {"raw": "1001 1001 1001 1001", "status": "present"}, + "Groups": {"raw": "", "status": "present"}, + "NoNewPrivs": {"raw": "1", "status": "present"}, + "Seccomp": {"raw": "2", "status": "present"}, + "Seccomp_filters": {"raw": "1", "status": "present"}, + "Uid": {"raw": "1001 1001 1001 1001", "status": "present"}, + } + status_text = "".join(f"{name}:\t{value['raw']}\n" for name, value in state_fields.items()) + status_snapshot = { + "bytes": len(status_text.encode("ascii")), + "content": status_text, + "content_encoding": "ascii", + "content_sha256": hashlib.sha256(status_text.encode("ascii")).hexdigest(), + "gid": 1001, + "kind": "file", + "mode": 0o444, + "path": "/proc/self/status", + "status": "present_readable", + "uid": 1001, + } + mount_text = "1 0 0:1 / / rw,relatime shared:1 - rootfs rootfs rw\n" + mount_snapshot = { + "bytes": len(mount_text.encode("ascii")), + "content": mount_text, + "content_encoding": "ascii", + "content_sha256": hashlib.sha256(mount_text.encode("ascii")).hexdigest(), + "gid": 1001, + "kind": "file", + "mode": 0o444, + "path": "/proc/self/mountinfo", + "status": "present_readable", + "uid": 1001, + } + present_zero = { + "bytes": 2, + "content": "0\n", + "content_encoding": "ascii", + "content_sha256": hashlib.sha256(b"0\n").hexdigest(), + "gid": 0, + "kind": "file", + "mode": 0o444, + "path": HOST_POLICY_CONTROL_PATHS[0], + "status": "present_readable", + "uid": 0, + } + present_false = { + **present_zero, + "bytes": 2, + "content": "N\n", + "content_sha256": hashlib.sha256(b"N\n").hexdigest(), + "path": HOST_POLICY_CONTROL_PATHS[4], + } + absent = {"path": HOST_POLICY_CONTROL_PATHS[1], "status": "absent"} + unreadable = { + "error": "Permission denied", + "error_type": "PermissionError", + "errno": 13, + "operation": "read", + "path": HOST_POLICY_CONTROL_PATHS[2], + "status": "unreadable", + } + directory_entries = ["apparmor", "lsm"] + directory = { + "entries": directory_entries, + "entries_sha256": hashlib.sha256(bytes_for(directory_entries)).hexdigest(), + "gid": 0, + "kind": "directory", + "mode": 0o555, + "path": HOST_POLICY_CONTROL_PATHS[3], + "status": "present_readable", + "uid": 0, + } + controls = {} + for path in HOST_POLICY_CONTROL_PATHS: + controls[path] = {"path": path, "status": "absent"} + controls[HOST_POLICY_CONTROL_PATHS[0]] = present_zero + controls[HOST_POLICY_CONTROL_PATHS[1]] = absent + controls[HOST_POLICY_CONTROL_PATHS[2]] = unreadable + controls[HOST_POLICY_CONTROL_PATHS[3]] = directory + controls[HOST_POLICY_CONTROL_PATHS[4]] = present_false + synthetic = { + "captured_state": valid, + "controls": controls, + "format": "dsv41-host-policy-diagnostic", + "proc_self_status": { + "fields": state_fields, + "snapshot": status_snapshot, + }, + "root_mount_propagation": { + "root_mounts": root_mount_evidence(mount_snapshot), + "snapshot": mount_snapshot, + }, + "uname": { + "machine": "aarch64", "nodename": "runner", "release": "6.11", + "status": "available", "sysname": "Linux", "version": "#1", + }, + "version": 1, + } + validate_host_policy_diagnostic(synthetic, valid) + unavailable_status = { + **synthetic, + "proc_self_status": { + "fields": { + name: {"status": "unavailable"} + for name in HOST_POLICY_STATUS_FIELDS + }, + "snapshot": {"path": "/proc/self/status", "status": "absent"}, + }, + "uname": { + "error": "unavailable", "error_type": "OSError", + "errno": None, "status": "unavailable", + }, + } + validate_host_policy_diagnostic(unavailable_status, valid) + partial_status = { + **synthetic, + "proc_self_status": { + "fields": { + **state_fields, + "Seccomp_filters": {"status": "absent"}, + }, + "snapshot": status_snapshot, + }, + } + validate_host_policy_diagnostic(partial_status, valid) + unavailable_mount = { + **synthetic, + "root_mount_propagation": { + "root_mounts": [], + "snapshot": {"path": "/proc/self/mountinfo", "status": "absent"}, + }, + } + validate_host_policy_diagnostic(unavailable_mount, valid) + for mutation in ( + {"format": "wrong"}, + {"version": 2}, + {"captured_state": {**valid, "no_new_privs": 0}}, + {"controls": {**controls, "/wrong": {"path": "/wrong", "status": "absent"}}}, + {"proc_self_status": { + "fields": {**state_fields, "NoNewPrivs": {"raw": "0", "status": "present"}}, + "snapshot": status_snapshot, + }}, + {"root_mount_propagation": {"root_mounts": [], "snapshot": mount_snapshot}}): + changed = dict(synthetic) + changed.update(mutation) + rejected(lambda changed=changed: validate_host_policy_diagnostic(changed, valid)) + for invalid, expected_path in ( + ({**present_zero, "content": "1\n"}, HOST_POLICY_CONTROL_PATHS[0]), + ({**present_zero, "bytes": 1}, HOST_POLICY_CONTROL_PATHS[0]), + ({**present_zero, "content_sha256": "0" * 64}, HOST_POLICY_CONTROL_PATHS[0]), + ({**absent, "content": ""}, HOST_POLICY_CONTROL_PATHS[1]), + ({**unreadable, "error": ""}, HOST_POLICY_CONTROL_PATHS[2]), + ({**unreadable, "content": ""}, HOST_POLICY_CONTROL_PATHS[2]), + ({**directory, "entries_sha256": "0" * 64}, HOST_POLICY_CONTROL_PATHS[3]), + ({"path": HOST_POLICY_CONTROL_PATHS[0], "status": "invalid"}, HOST_POLICY_CONTROL_PATHS[0]), + ({**present_zero, "path": "/wrong"}, HOST_POLICY_CONTROL_PATHS[0])): + rejected(lambda invalid=invalid, expected_path=expected_path: validate_snapshot(invalid, expected_path)) root = Path(output).resolve().parent / "gate-self-test" root.mkdir(mode=0o700) with tempfile.TemporaryDirectory(dir=root) as temp: @@ -559,6 +1098,7 @@ jobs: rejected(lambda: verify_digest(marker, digest)) Path(output).write_text(json.dumps({ "environment": "PASS", "exclusive_and_tamper": "PASS", + "host_policy_diagnostic": "PASS", "interpreter": "PASS", "secure_ancestry": "PASS", "state_matrix": "PASS", }, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") @@ -618,6 +1158,9 @@ jobs: path, digest, before = preflight() state = GATE.current_state() GATE.validate_state(state, CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) + diagnostic = GATE.validate_host_policy_file( + before["host_policy_diagnostic"], state, + Path(CONFIG["paths"]["evidence"]) / "host-policy-diagnostic.json") interpreter = GATE.validate_interpreter(CONFIG) if any(os.environ.get(name) for name in GATE.FORBIDDEN): raise RuntimeError("target process has a forbidden loader environment") @@ -628,6 +1171,7 @@ jobs: raise RuntimeError("target process differs from preflight") return { "active_forbidden_loader_environment": [], + "host_policy_diagnostic": diagnostic, "interpreter": interpreter, "preflight_path": str(path), "preflight_sha256": digest, @@ -769,10 +1313,25 @@ jobs: f"marker counts are invalid: entry={entries} duplicate={duplicates} " f"target_skip={target_skips} skips={total_skips}") - def validate_marker(status, paths): + def validate_preflight_diagnostic(paths): + evidence = Path(paths["evidence"]) + preflight_path = evidence / "runner-preflight.json" + preflight_sha = hashlib.sha256(preflight_path.read_bytes()).hexdigest() + preflight = json.loads(preflight_path.read_text(encoding="ascii")) + GATE.validate_state( + preflight["state"], CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) + diagnostic = GATE.validate_host_policy_file( + preflight["host_policy_diagnostic"], preflight["state"], + evidence / "host-policy-diagnostic.json") + return diagnostic, preflight_path, preflight_sha, preflight + + def validate_marker(status, paths, preflight_evidence=None): evidence = Path(paths["evidence"]) marker_dir = Path(paths["marker"]) log = (Path(paths["log"]) / "test-deepseek41-trace-ctest.log").read_text(errors="replace") + diagnostic, preflight_path, preflight_sha, preflight = ( + validate_preflight_diagnostic(paths) + if preflight_evidence is None else preflight_evidence) validate_outer( status, junit_values(evidence / "test-deepseek41-trace-ctest.xml"), @@ -780,11 +1339,6 @@ jobs: ) metadata = json.loads((evidence / "test-deepseek41-trace-ctest-metadata.json").read_text()) bindings = expected_bindings(metadata) - preflight_path = evidence / "runner-preflight.json" - preflight_sha = hashlib.sha256(preflight_path.read_bytes()).hexdigest() - preflight = json.loads(preflight_path.read_text(encoding="ascii")) - GATE.validate_state( - preflight["state"], CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) if preflight["environment_names"] != CONFIG["environment_allowlist"]: raise RuntimeError("preflight environment allowlist differs") if preflight["ctest"] != str(Path(CONFIG["ctest"]).resolve(strict=True)): @@ -816,6 +1370,7 @@ jobs: "event": "test_method_entry", "function": TARGET_METHOD, "gate_sha256": os.environ["DSV41_CI_GATE_SHA256"], + "host_policy_diagnostic": diagnostic, "hook_sha256": os.environ["DSV41_CI_HOOK_SHA256"], "interpreter": preflight["interpreter"], "preflight_path": str(preflight_path), @@ -850,6 +1405,7 @@ jobs: "ctest_execution_count": 1, "duplicate_entry_count": 0, "forbidden_loader_environment": [], + "host_policy_diagnostic": diagnostic, "junit": {"disabled": 0, "failures": 0, "skipped": 0, "tests": 1}, "preflight_sha256": preflight_sha, "result": "PASS", @@ -917,10 +1473,14 @@ jobs: }, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") def validate(status, output): + preflight_evidence = None try: - receipt = validate_marker(status, CONFIG["paths"]) + preflight_evidence = validate_preflight_diagnostic(CONFIG["paths"]) + receipt = validate_marker(status, CONFIG["paths"], preflight_evidence) except Exception as error: receipt = {"error": str(error), "result": "FAIL"} + if preflight_evidence is not None: + receipt["host_policy_diagnostic"] = preflight_evidence[0] GATE.write_exclusive(Path(output), receipt) raise GATE.write_exclusive(Path(output), receipt)