diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 77fe7dbd3aa7..f385499a9690 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -94,12 +94,26 @@ jobs: folder: llama.cpp hf_bucket: ggml-org/cache + - name: Python setup + id: setup_python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + pip-install: -r tools/server/tests/requirements.txt + - name: Build id: cmake_build run: | cmake -B build \ - -DGGML_SCHED_NO_REALLOC=ON - cmake --build build --config Release -j $(nproc) --target llama-server + -DGGML_SCHED_NO_REALLOC=ON \ + -DLLAMA_OPENSSL=OFF \ + -DPython3_EXECUTABLE="$(command -v python3)" + cmake --build build --config Release -j $(nproc) --target \ + llama-server \ + llama-deepseek-v41-trace \ + llama-deepseek-v41-containment-helper \ + test-deepseek41-trace-manifest \ + test-deepseek41-trace-injected - name: ccache-buckets-save if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} @@ -113,12 +127,1429 @@ jobs: hf_bucket: ggml-org/cache save: true - - name: Python setup - id: setup_python - uses: actions/setup-python@v6 + - name: DeepSeek V4.1 trace CTest + id: deepseek_v41_trace_ctest + run: | + set -o pipefail + ci_root="$PWD/build/deepseek-v41-trace-ci" + rm -rf "$ci_root" + install -d -m 0700 \ + "$ci_root" \ + "$ci_root/tmp" \ + "$ci_root/hook" \ + "$ci_root/marker" \ + "$ci_root/log" \ + "$ci_root/evidence" + runner_uid="$(id -u)" + runner_gid="$(id -g)" + runner_cap_bnd="$(awk '/^CapBnd:/ { print $2 }' /proc/self/status)" + selected_interpreter="$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.executable).resolve(strict=True))')" + ctest_path="$(command -v ctest)" + if [[ "$runner_uid" == "0" || "$runner_gid" == "0" || ! "$runner_cap_bnd" =~ ^[0-9A-Fa-f]+$ || "$selected_interpreter" != /* || "$ctest_path" != /* ]]; then + echo "hosted runner identity or executable path is invalid" >&2 + exit 1 + fi + export DSV41_CI_ROOT="$ci_root" + export DSV41_CI_RUNNER_UID="$runner_uid" + export DSV41_CI_RUNNER_GID="$runner_gid" + export DSV41_CI_RUNNER_CAP_BND="$runner_cap_bnd" + export DSV41_CI_SELECTED_INTERPRETER="$selected_interpreter" + export DSV41_CI_CTEST="$(realpath "$ctest_path")" + ctest --test-dir build --show-only=json-v1 -R '^test-deepseek41-trace$' > "$ci_root/evidence/test-deepseek41-trace-ctest-metadata.json" + python3 - <<'PY' + import ast + import hashlib + import json + import os + import subprocess + import sys + from pathlib import Path + + root = Path(os.environ["DSV41_CI_ROOT"]).resolve(strict=True) + evidence = root / "evidence" + workspace = Path.cwd().resolve(strict=True) + build = (workspace / "build").resolve(strict=True) + metadata_path = evidence / "test-deepseek41-trace-ctest-metadata.json" + metadata = json.loads(metadata_path.read_text()) + tests = metadata.get("tests", []) + if len(tests) != 1 or tests[0].get("name") != "test-deepseek41-trace": + raise SystemExit(f"expected exactly one test-deepseek41-trace selector match, got {len(tests)}") + command = tests[0].get("command", []) + if not command or not isinstance(command[0], str) or not Path(command[0]).is_absolute(): + raise SystemExit("test-deepseek41-trace interpreter is absent or relative") + registered_interpreter = Path(command[0]).resolve(strict=True) + selected_interpreter = Path(sys.executable).resolve(strict=True) + if registered_interpreter != selected_interpreter: + raise SystemExit( + f"test-deepseek41-trace interpreter mismatch: " + f"registered={registered_interpreter} selected={selected_interpreter}") + registered_version = subprocess.check_output( + [str(registered_interpreter), "-c", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"], + text=True, + ).strip() + if registered_version != "3.11" or sys.version_info[:2] != (3, 11): + raise SystemExit( + f"test-deepseek41-trace requires Python 3.11: " + f"registered={registered_version} selected={sys.version_info.major}.{sys.version_info.minor}") + (evidence / "test-deepseek41-trace-python.json").write_text( + json.dumps({ + "registered_interpreter": str(registered_interpreter), + "registered_version": registered_version, + "selected_interpreter": str(selected_interpreter), + "selected_version": f"{sys.version_info.major}.{sys.version_info.minor}", + }, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + properties = { + item["name"]: item["value"] + for item in tests[0].get("properties", []) + } + environment = properties.get("ENVIRONMENT", []) + if isinstance(environment, str): + environment = environment.split(";") + required = ( + "DSV41_NATIVE_TRACE_BINARY=", + "DSV41_NATIVE_CONTAINMENT_HELPER=", + "DSV41_NATIVE_MANIFEST_BINARY=", + "DSV41_NATIVE_INJECT_LIBRARY=", + ) + missing = [ + name + for name in required + if not any(value.startswith(name) for value in environment) + ] + if missing: + raise SystemExit(f"test-deepseek41-trace environment is incomplete: {missing}") + source = workspace / "tools/deepseek-v41-trace/trace_format.py" + parsed = ast.parse(source.read_text(encoding="ascii")) + forbidden = None + for node in parsed.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "FORBIDDEN_LOADER_ENVIRONMENT" + for target in node.targets): + forbidden = ast.literal_eval(node.value) + break + expected_forbidden = ( + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_IMAGE_SUFFIX", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "DYLD_ROOT_PATH", + "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", + "GGML_BACKEND_PATH", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + ) + if forbidden != expected_forbidden: + raise SystemExit("trace forbidden loader environment changed") + forbidden_path = evidence / "forbidden-loader-environment.json" + forbidden_path.write_text( + json.dumps(list(forbidden), separators=(",", ":")) + "\n", + encoding="ascii", + ) + config = { + "ctest": str(Path(os.environ["DSV41_CI_CTEST"]).resolve(strict=True)), + "forbidden_path": str(forbidden_path), + "forbidden_sha256": hashlib.sha256(forbidden_path.read_bytes()).hexdigest(), + "environment_allowlist": sorted(( + "DSV41_CI_CONFIG", + "DSV41_CI_CONFIG_SHA256", + "DSV41_CI_GATE", + "DSV41_CI_GATE_SHA256", + "DSV41_CI_HOOK_SHA256", + "HOME", + "LANG", + "LC_ALL", + "PATH", + "PYTHONDONTWRITEBYTECODE", + "PYTHONCOERCECLOCALE", + "PYTHONPATH", + "PYTHONUTF8", + "TMPDIR", + )), + "paths": { + "build": str(build), + "evidence": str(evidence), + "hook": str(root / "hook"), + "log": str(root / "log"), + "marker": str(root / "marker"), + "tmp": str(root / "tmp"), + "workspace": str(workspace), + }, + "registered_interpreter": str(registered_interpreter), + "runner_cap_bnd": int(os.environ["DSV41_CI_RUNNER_CAP_BND"], 16), + "runner_gid": int(os.environ["DSV41_CI_RUNNER_GID"]), + "runner_uid": int(os.environ["DSV41_CI_RUNNER_UID"]), + "selected_interpreter": str(selected_interpreter), + "source": str((workspace / "tests/test-deepseek41-trace.py").resolve(strict=True)), + } + (evidence / "fixture-config.json").write_text( + json.dumps(config, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + PY + cat > "$ci_root/evidence/fixture_gate.py" <<'PY' + import hashlib + import json + import os + import stat + import sys + import tempfile + import time + from pathlib import Path + + FORBIDDEN = ( + "DYLD_FALLBACK_FRAMEWORK_PATH", "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", "DYLD_IMAGE_SUFFIX", "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", "DYLD_ROOT_PATH", "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", "GGML_BACKEND_PATH", "LD_AUDIT", + "LD_LIBRARY_PATH", "LD_PRELOAD", + ) + HOST_POLICY_STATUS_FIELDS = ( + "Uid", "Gid", "Groups", "CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb", + "NoNewPrivs", "Seccomp", "Seccomp_filters", + ) + HOST_POLICY_CONTROL_PATHS = ( + "/proc/self/attr/current", + "/sys/module/apparmor/parameters/enabled", + "/proc/sys/kernel/unprivileged_userns_clone", + "/proc/sys/user/max_user_namespaces", + "/proc/sys/kernel/apparmor_restrict_unprivileged_userns", + "/sys/kernel/security/lsm", + "/sys/kernel/security/apparmor/profiles", + "/sys/kernel/security", + "/sys/kernel/security/apparmor", + "/sys/kernel/security/apparmor/features", + ) + + def bytes_for(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + def write_exclusive(path, value): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags, 0o600) + data = bytes_for(value) + try: + if os.write(descriptor, data) != len(data): + raise RuntimeError(f"short write for {path}") + os.fsync(descriptor) + finally: + os.close(descriptor) + return hashlib.sha256(data).hexdigest() + + def verify_digest(path, expected): + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + raise RuntimeError(f"SHA-256 mismatch for {path}") + return actual + + def read_status(text=None): + text = Path("/proc/self/status").read_text(encoding="ascii") if text is None else text + fields = {} + for line in text.splitlines(): + name, separator, value = line.partition(":") + if separator: + fields[name] = value.strip() + required = ("Uid", "Gid", "Groups", "CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb", "NoNewPrivs") + missing = [name for name in required if name not in fields] + if missing: + raise RuntimeError(f"/proc/self/status is incomplete: {missing}") + return { + "cap_amb": int(fields["CapAmb"], 16), + "cap_bnd": int(fields["CapBnd"], 16), + "cap_eff": int(fields["CapEff"], 16), + "cap_inh": int(fields["CapInh"], 16), + "cap_prm": int(fields["CapPrm"], 16), + "gid": [int(value) for value in fields["Gid"].split()], + "groups": [int(value) for value in fields["Groups"].split()], + "no_new_privs": int(fields["NoNewPrivs"]), + "uid": [int(value) for value in fields["Uid"].split()], + } + + def current_state(): + state = read_status() + state.update({ + "os_egid": os.getegid(), "os_euid": os.geteuid(), + "os_gid": os.getgid(), "os_groups": os.getgroups(), "os_uid": os.getuid(), + }) + return state + + def error_snapshot(path, operation, error, metadata=None): + result = { + "error": str(error), + "error_type": type(error).__name__, + "errno": getattr(error, "errno", None), + "operation": operation, + "path": str(path), + "status": "unreadable", + } + if metadata is not None: + result.update(metadata) + return result + + def path_metadata(path, info): + kind = "directory" if stat.S_ISDIR(info.st_mode) else "file" if stat.S_ISREG(info.st_mode) else "other" + return { + "gid": info.st_gid, + "kind": kind, + "mode": stat.S_IMODE(info.st_mode), + "uid": info.st_uid, + } + + def snapshot_path(value): + path = Path(value) + try: + info = path.lstat() + except FileNotFoundError: + return {"path": str(path), "status": "absent"} + except OSError as error: + return error_snapshot(path, "lstat", error) + metadata = path_metadata(path, info) + if metadata["kind"] == "directory": + try: + entries = sorted(os.listdir(path)) + except OSError as error: + return error_snapshot(path, "listdir", error, metadata) + encoded = bytes_for(entries) + return { + **metadata, + "entries": entries, + "entries_sha256": hashlib.sha256(encoded).hexdigest(), + "path": str(path), + "status": "present_readable", + } + if metadata["kind"] != "file": + return { + **metadata, + "path": str(path), + "status": "present_readable", + } + try: + data = path.read_bytes() + except OSError as error: + return error_snapshot(path, "read", error, metadata) + try: + content = data.decode("ascii") + encoding = "ascii" + except UnicodeDecodeError: + content = data.hex() + encoding = "hex" + return { + **metadata, + "bytes": len(data), + "content": content, + "content_encoding": encoding, + "content_sha256": hashlib.sha256(data).hexdigest(), + "path": str(path), + "status": "present_readable", + } + + def validate_snapshot(snapshot, expected_path): + if snapshot.get("path") != expected_path: + raise RuntimeError(f"host policy path differs: {snapshot.get('path')} != {expected_path}") + status = snapshot.get("status") + if status == "absent": + if set(snapshot) != {"path", "status"}: + raise RuntimeError(f"absent host policy control has extra fields: {expected_path}") + return + if status == "unreadable": + allowed = { + "error", "error_type", "errno", "operation", "path", "status", + "gid", "kind", "mode", "uid", + } + if not set(snapshot) <= allowed: + raise RuntimeError(f"unreadable host policy control has extra fields: {expected_path}") + if not isinstance(snapshot.get("operation"), str) or not snapshot["operation"]: + raise RuntimeError(f"unreadable host policy control has no operation: {expected_path}") + if not isinstance(snapshot.get("error_type"), str) or not snapshot["error_type"]: + raise RuntimeError(f"unreadable host policy control has no error type: {expected_path}") + if not isinstance(snapshot.get("error"), str) or not snapshot["error"]: + raise RuntimeError(f"unreadable host policy control has no error: {expected_path}") + if snapshot.get("errno") is not None and not isinstance(snapshot["errno"], int): + raise RuntimeError(f"unreadable host policy control errno is invalid: {expected_path}") + metadata = [name in snapshot for name in ("mode", "uid", "gid", "kind")] + if any(metadata) and not all(metadata): + raise RuntimeError(f"unreadable host policy control metadata is incomplete: {expected_path}") + if all(metadata): + if snapshot["kind"] not in ("directory", "file", "other"): + raise RuntimeError(f"unreadable host policy control kind is invalid: {expected_path}") + for name in ("mode", "uid", "gid"): + if not isinstance(snapshot[name], int) or snapshot[name] < 0: + raise RuntimeError(f"unreadable host policy control {name} is invalid: {expected_path}") + return + if status != "present_readable": + raise RuntimeError(f"host policy control status is invalid: {expected_path}") + for name in ("mode", "uid", "gid"): + if not isinstance(snapshot.get(name), int) or snapshot[name] < 0: + raise RuntimeError(f"host policy control {name} is invalid: {expected_path}") + kind = snapshot.get("kind") + if kind == "directory": + expected = { + "entries", "entries_sha256", "gid", "kind", "mode", "path", "status", "uid", + } + if set(snapshot) != expected: + raise RuntimeError(f"host policy directory fields differ: {expected_path}") + entries = snapshot.get("entries") + if not isinstance(entries, list) or entries != sorted(entries) or not all( + isinstance(item, str) for item in entries): + raise RuntimeError(f"host policy directory entries are invalid: {expected_path}") + if snapshot.get("entries_sha256") != hashlib.sha256(bytes_for(entries)).hexdigest(): + raise RuntimeError(f"host policy directory entry hash differs: {expected_path}") + return + if kind == "other": + if set(snapshot) != {"gid", "kind", "mode", "path", "status", "uid"}: + raise RuntimeError(f"host policy other fields differ: {expected_path}") + return + if kind != "file": + raise RuntimeError(f"host policy control kind is invalid: {expected_path}") + expected = { + "bytes", "content", "content_encoding", "content_sha256", + "gid", "kind", "mode", "path", "status", "uid", + } + if set(snapshot) != expected: + raise RuntimeError(f"host policy file fields differ: {expected_path}") + encoding = snapshot.get("content_encoding") + content = snapshot.get("content") + if not isinstance(content, str) or encoding not in ("ascii", "hex"): + raise RuntimeError(f"host policy control content is invalid: {expected_path}") + try: + data = content.encode("ascii") if encoding == "ascii" else bytes.fromhex(content) + except (UnicodeEncodeError, ValueError) as error: + raise RuntimeError(f"host policy control encoding is invalid: {expected_path}") from error + if snapshot.get("bytes") != len(data): + raise RuntimeError(f"host policy control byte count differs: {expected_path}") + if snapshot.get("content_sha256") != hashlib.sha256(data).hexdigest(): + raise RuntimeError(f"host policy control hash differs: {expected_path}") + + def snapshot_ascii(snapshot): + validate_snapshot(snapshot, snapshot.get("path")) + if snapshot.get("status") != "present_readable" or snapshot.get("kind") != "file": + return None + if snapshot.get("content_encoding") != "ascii": + raise RuntimeError(f"host policy control is not ASCII: {snapshot['path']}") + return snapshot["content"] + + def status_field_evidence(status_snapshot): + text = snapshot_ascii(status_snapshot) + if text is None: + return { + name: {"status": "unavailable"} + for name in HOST_POLICY_STATUS_FIELDS + } + fields = {} + for line in text.splitlines(): + name, separator, value = line.partition(":") + if separator: + fields[name] = value.strip() + return { + name: {"raw": fields[name], "status": "present"} if name in fields else {"status": "absent"} + for name in HOST_POLICY_STATUS_FIELDS + } + + def root_mount_evidence(mountinfo_snapshot): + text = snapshot_ascii(mountinfo_snapshot) + if text is None: + return [] + result = [] + for line in text.splitlines(): + before, separator, after = line.partition(" - ") + fields = before.split() + suffix = after.split() + if not separator or len(fields) < 6 or len(suffix) < 3 or fields[4] != "/": + continue + result.append({ + "filesystem_type": suffix[0], + "major_minor": fields[2], + "mount_id": int(fields[0]), + "mount_options": fields[5], + "mount_point": fields[4], + "mount_source": suffix[1], + "optional_fields": fields[6:], + "parent_id": int(fields[1]), + "raw_line": line, + "root": fields[3], + "super_options": suffix[2:], + }) + return result + + def capture_host_policy_diagnostic(state): + try: + uname = os.uname() + uname_evidence = { + "machine": uname.machine, + "nodename": uname.nodename, + "release": uname.release, + "status": "available", + "sysname": uname.sysname, + "version": uname.version, + } + except OSError as error: + uname_evidence = { + "error": str(error), + "error_type": type(error).__name__, + "errno": error.errno, + "status": "unavailable", + } + status_snapshot = snapshot_path("/proc/self/status") + mountinfo_snapshot = snapshot_path("/proc/self/mountinfo") + diagnostic = { + "captured_state": state, + "controls": { + path: snapshot_path(path) + for path in HOST_POLICY_CONTROL_PATHS + }, + "format": "dsv41-host-policy-diagnostic", + "proc_self_status": { + "fields": status_field_evidence(status_snapshot), + "snapshot": status_snapshot, + }, + "root_mount_propagation": { + "root_mounts": root_mount_evidence(mountinfo_snapshot), + "snapshot": mountinfo_snapshot, + }, + "uname": uname_evidence, + "version": 1, + } + validate_host_policy_diagnostic(diagnostic, state) + return diagnostic + + def parse_decimal_field(fields, name): + item = fields[name] + if item.get("status") == "absent": + return None + if item.get("status") != "present" or not isinstance(item.get("raw"), str): + raise RuntimeError(f"host policy status field is invalid: {name}") + try: + return int(item["raw"], 10) + except ValueError as error: + raise RuntimeError(f"host policy status field is not decimal: {name}") from error + + def parse_vector_field(fields, name): + item = fields[name] + if item.get("status") != "present" or not isinstance(item.get("raw"), str): + raise RuntimeError(f"host policy status field is invalid: {name}") + try: + return [int(value, 10) for value in item["raw"].split()] + except ValueError as error: + raise RuntimeError(f"host policy status vector is invalid: {name}") from error + + def parse_hex_field(fields, name): + item = fields[name] + if item.get("status") != "present" or not isinstance(item.get("raw"), str): + raise RuntimeError(f"host policy status field is invalid: {name}") + try: + return int(item["raw"], 16) + except ValueError as error: + raise RuntimeError(f"host policy status field is not hexadecimal: {name}") from error + + def validate_host_policy_diagnostic(diagnostic, state): + if diagnostic.get("format") != "dsv41-host-policy-diagnostic" or diagnostic.get("version") != 1: + raise RuntimeError("host policy diagnostic schema differs") + if diagnostic.get("captured_state") != state: + raise RuntimeError("host policy diagnostic state differs") + uname = diagnostic.get("uname", {}) + if uname.get("status") == "available": + if not all(isinstance(uname.get(name), str) for name in ( + "sysname", "nodename", "release", "version", "machine")): + raise RuntimeError("host policy uname evidence is invalid") + elif uname.get("status") == "unavailable": + if not isinstance(uname.get("error_type"), str) or not isinstance(uname.get("error"), str): + raise RuntimeError("host policy uname error is invalid") + else: + raise RuntimeError("host policy uname status is invalid") + status = diagnostic.get("proc_self_status", {}) + snapshot = status.get("snapshot", {}) + validate_snapshot(snapshot, "/proc/self/status") + fields = status.get("fields") + if not isinstance(fields, dict) or set(fields) != set(HOST_POLICY_STATUS_FIELDS): + raise RuntimeError("host policy status fields differ") + if snapshot.get("status") != "present_readable": + if any(value != {"status": "unavailable"} for value in fields.values()): + raise RuntimeError("unavailable host policy status has values") + else: + if parse_vector_field(fields, "Uid") != state["uid"]: + raise RuntimeError("host policy Uid differs") + if parse_vector_field(fields, "Gid") != state["gid"]: + raise RuntimeError("host policy Gid differs") + if parse_vector_field(fields, "Groups") != state["groups"]: + raise RuntimeError("host policy Groups differs") + for field, name in ( + ("CapInh", "cap_inh"), ("CapPrm", "cap_prm"), ("CapEff", "cap_eff"), + ("CapBnd", "cap_bnd"), ("CapAmb", "cap_amb")): + if parse_hex_field(fields, field) != state[name]: + raise RuntimeError(f"host policy {field} differs") + if parse_decimal_field(fields, "NoNewPrivs") != state["no_new_privs"]: + raise RuntimeError("host policy NoNewPrivs differs") + for name in ("Seccomp", "Seccomp_filters"): + parse_decimal_field(fields, name) + controls = diagnostic.get("controls") + if not isinstance(controls, dict) or set(controls) != set(HOST_POLICY_CONTROL_PATHS): + raise RuntimeError("host policy control paths differ") + for path in HOST_POLICY_CONTROL_PATHS: + validate_snapshot(controls[path], path) + mount = diagnostic.get("root_mount_propagation", {}) + validate_snapshot(mount.get("snapshot", {}), "/proc/self/mountinfo") + roots = mount.get("root_mounts") + if not isinstance(roots, list): + raise RuntimeError("host policy root mounts are invalid") + if mount["snapshot"].get("status") == "present_readable" and not roots: + raise RuntimeError("host policy root mount is absent") + if mount["snapshot"].get("status") != "present_readable" and roots: + raise RuntimeError("unavailable host policy mount has root entries") + for item in roots: + if item.get("mount_point") != "/" or not isinstance(item.get("raw_line"), str): + raise RuntimeError("host policy root mount entry is invalid") + + def validate_host_policy_file(binding, state, expected_path): + path = Path(binding.get("path", "")) + if not path.is_absolute() or path.resolve(strict=True) != path or path != Path(expected_path): + raise RuntimeError("host policy diagnostic path is invalid") + info = path.lstat() + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + raise RuntimeError("host policy diagnostic is not a direct file") + if stat.S_IMODE(info.st_mode) != 0o444 or binding.get("mode") != 0o444: + raise RuntimeError("host policy diagnostic mode differs") + digest = verify_digest(path, binding.get("sha256")) + diagnostic = json.loads(path.read_text(encoding="ascii")) + validate_host_policy_diagnostic(diagnostic, state) + if binding.get("format") != diagnostic["format"] or binding.get("version") != diagnostic["version"]: + raise RuntimeError("host policy diagnostic binding schema differs") + return { + "format": diagnostic["format"], + "mode": 0o444, + "path": str(path), + "sha256": digest, + "version": diagnostic["version"], + } + + def validate_state(state, uid, gid, cap_bnd): + if uid <= 0 or gid <= 0: + raise RuntimeError("runner identity is privileged") + if state["uid"] != [uid] * 4 or state["gid"] != [gid] * 4: + raise RuntimeError("live UID or GID differs") + if (state["os_uid"], state["os_euid"], state["os_gid"], state["os_egid"]) != (uid, uid, gid, gid): + raise RuntimeError("runtime UID or GID differs") + if state["groups"] or state["os_groups"]: + raise RuntimeError("supplementary groups are not empty") + for name in ("cap_inh", "cap_prm", "cap_eff", "cap_amb"): + if state[name] != 0: + raise RuntimeError(f"{name} is not zero") + if state["cap_bnd"] != cap_bnd: + raise RuntimeError("CapBnd changed") + if state["no_new_privs"] != 1: + raise RuntimeError("NoNewPrivs is not one") + + def validate_interpreter_values(registered, selected, executable, registered_version, selected_version): + if not all(Path(value).is_absolute() for value in (registered, selected, executable)): + raise RuntimeError("Python path is not absolute") + if registered != selected or registered != executable: + raise RuntimeError("Python identity differs") + if registered_version != "3.11" or selected_version != "3.11": + raise RuntimeError("Python is not version 3.11") + + def validate_interpreter(config): + registered = str(Path(config["registered_interpreter"]).resolve(strict=True)) + selected = str(Path(config["selected_interpreter"]).resolve(strict=True)) + executable = str(Path(sys.executable).resolve(strict=True)) + version = f"{sys.version_info.major}.{sys.version_info.minor}" + validate_interpreter_values(registered, selected, executable, version, version) + return {"executable": executable, "registered": registered, "selected": selected, "version": version} + + def load_forbidden(config): + path = Path(config["forbidden_path"]).resolve(strict=True) + verify_digest(path, config["forbidden_sha256"]) + values = tuple(json.loads(path.read_text(encoding="ascii"))) + if values != FORBIDDEN: + raise RuntimeError("forbidden loader list differs") + return values + + def validate_environment(environment, config): + names = sorted(environment) + if names != config["environment_allowlist"]: + raise RuntimeError(f"environment allowlist differs: {names}") + active = sorted(name for name in FORBIDDEN if environment.get(name)) + if active: + raise RuntimeError(f"forbidden loader environment is active: {active}") + return names + + def directory_receipts(workspace, build, direct_paths, uid): + workspace, build = Path(workspace), Path(build) + direct_paths = [Path(value) for value in direct_paths] + for path in (workspace, build, *direct_paths): + if not path.is_absolute() or path.resolve(strict=True) != path: + raise RuntimeError(f"directory is not canonical: {path}") + if build != workspace and workspace not in build.parents: + raise RuntimeError("build is outside workspace") + receipts = {} + for target in (build, *direct_paths): + if target != build and build not in target.parents: + raise RuntimeError(f"staging path is outside build: {target}") + current = workspace + candidates = [current] + for part in target.relative_to(workspace).parts: + current /= part + candidates.append(current) + for candidate in candidates: + if str(candidate) in receipts: + continue + info = candidate.lstat() + mode = stat.S_IMODE(info.st_mode) + if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode): + raise RuntimeError(f"staging ancestry is not a direct directory: {candidate}") + if info.st_uid != uid or mode & 0o022: + raise RuntimeError(f"staging ancestry owner or mode is invalid: {candidate}") + receipts[str(candidate)] = { + "device": info.st_dev, "gid": info.st_gid, "inode": info.st_ino, + "mode": mode, "uid": info.st_uid, + } + for path in direct_paths: + if receipts[str(path)]["mode"] != 0o700: + raise RuntimeError(f"staging directory mode is not 0700: {path}") + return receipts + + def load_config(): + path = Path(os.environ["DSV41_CI_CONFIG"]).resolve(strict=True) + verify_digest(path, os.environ["DSV41_CI_CONFIG_SHA256"]) + return json.loads(path.read_text(encoding="ascii")) + + def launch(): + config = load_config() + load_forbidden(config) + environment_names = validate_environment(os.environ, config) + state = current_state() + validate_state(state, config["runner_uid"], config["runner_gid"], config["runner_cap_bnd"]) + interpreter = validate_interpreter(config) + paths = config["paths"] + direct = [paths[name] for name in ("evidence", "hook", "log", "marker", "tmp")] + receipts = directory_receipts(paths["workspace"], paths["build"], direct, config["runner_uid"]) + ctest = str(Path(config["ctest"]).resolve(strict=True)) + if not os.access(ctest, os.X_OK): + raise RuntimeError("CTest is not executable") + diagnostic_path = Path(paths["evidence"]) / "host-policy-diagnostic.json" + diagnostic = capture_host_policy_diagnostic(state) + diagnostic_sha256 = write_exclusive(diagnostic_path, diagnostic) + diagnostic_path.chmod(0o444) + diagnostic_binding = validate_host_policy_file({ + "format": diagnostic["format"], + "mode": 0o444, + "path": str(diagnostic_path), + "sha256": diagnostic_sha256, + "version": diagnostic["version"], + }, state, diagnostic_path) + preflight = { + "ctest": ctest, "environment_names": environment_names, + "format": "dsv41-ci-runner-preflight", "interpreter": interpreter, + "host_policy_diagnostic": diagnostic_binding, + "pid": os.getpid(), "ppid": os.getppid(), "secure_directories": receipts, + "state": state, "timestamp_monotonic_ns": time.monotonic_ns(), "version": 1, + } + path = Path(paths["evidence"]) / "runner-preflight.json" + digest = write_exclusive(path, preflight) + path.chmod(0o444) + environment = dict(os.environ) + environment["DSV41_CI_PREFLIGHT"] = str(path) + environment["DSV41_CI_PREFLIGHT_SHA256"] = digest + os.execve(ctest, [ + ctest, "--test-dir", paths["build"], "-R", "^test-deepseek41-trace$", + "--output-on-failure", "--verbose", "--no-tests=error", + "--output-junit", str(Path(paths["evidence"]) / "test-deepseek41-trace-ctest.xml"), + ], environment) + + def rejected(function): + try: + function() + except Exception: + return + raise RuntimeError("negative fixture was accepted") + + def self_test(output): + parsed = read_status( + "Uid:\t1001 1001 1001 1001\n" + "Gid:\t1001 1001 1001 1001\n" + "Groups:\t\n" + "CapInh:\t0000000000000000\n" + "CapPrm:\t0000000000000000\n" + "CapEff:\t0000000000000000\n" + "CapBnd:\t000000000000000f\n" + "CapAmb:\t0000000000000000\n" + "NoNewPrivs:\t1\n") + expected_parsed = { + "cap_amb": 0, "cap_bnd": 15, "cap_eff": 0, "cap_inh": 0, "cap_prm": 0, + "gid": [1001] * 4, "groups": [], "no_new_privs": 1, "uid": [1001] * 4, + } + if parsed != expected_parsed: + raise RuntimeError("accepted /proc/self/status fixture differs") + valid = { + "cap_amb": 0, "cap_bnd": 15, "cap_eff": 0, "cap_inh": 0, "cap_prm": 0, + "gid": [1001] * 4, "groups": [], "no_new_privs": 1, + "os_egid": 1001, "os_euid": 1001, "os_gid": 1001, + "os_groups": [], "os_uid": 1001, "uid": [1001] * 4, + } + validate_state(valid, 1001, 1001, 15) + for mutation in ( + {"groups": [5]}, {"os_groups": [5]}, + {"uid": [1002] * 4}, {"gid": [1002] * 4}, + {"cap_inh": 1}, {"cap_prm": 1}, {"cap_eff": 1}, {"cap_amb": 1}, + {"cap_bnd": 14}, {"no_new_privs": 0}): + changed = dict(valid) + changed.update(mutation) + rejected(lambda changed=changed: validate_state(changed, 1001, 1001, 15)) + validate_interpreter_values("/python3.11", "/python3.11", "/python3.11", "3.11", "3.11") + rejected(lambda: validate_interpreter_values("python3.11", "/python3.11", "/python3.11", "3.11", "3.11")) + rejected(lambda: validate_interpreter_values("/python3.11", "/other", "/python3.11", "3.11", "3.11")) + rejected(lambda: validate_interpreter_values("/python3.11", "/python3.11", "/python3.11", "3.12", "3.11")) + config = {"environment_allowlist": ["HOME"]} + validate_environment({"HOME": "/home/runner"}, config) + rejected(lambda: validate_environment({"HOME": "/home/runner", "LD_LIBRARY_PATH": "/lib"}, config)) + rejected(lambda: validate_environment( + {"HOME": "/home/runner", "LD_LIBRARY_PATH": "/lib"}, + {"environment_allowlist": ["HOME", "LD_LIBRARY_PATH"]})) + state_fields = { + "CapAmb": {"raw": "0000000000000000", "status": "present"}, + "CapBnd": {"raw": "000000000000000f", "status": "present"}, + "CapEff": {"raw": "0000000000000000", "status": "present"}, + "CapInh": {"raw": "0000000000000000", "status": "present"}, + "CapPrm": {"raw": "0000000000000000", "status": "present"}, + "Gid": {"raw": "1001 1001 1001 1001", "status": "present"}, + "Groups": {"raw": "", "status": "present"}, + "NoNewPrivs": {"raw": "1", "status": "present"}, + "Seccomp": {"raw": "2", "status": "present"}, + "Seccomp_filters": {"raw": "1", "status": "present"}, + "Uid": {"raw": "1001 1001 1001 1001", "status": "present"}, + } + status_text = "".join(f"{name}:\t{value['raw']}\n" for name, value in state_fields.items()) + status_snapshot = { + "bytes": len(status_text.encode("ascii")), + "content": status_text, + "content_encoding": "ascii", + "content_sha256": hashlib.sha256(status_text.encode("ascii")).hexdigest(), + "gid": 1001, + "kind": "file", + "mode": 0o444, + "path": "/proc/self/status", + "status": "present_readable", + "uid": 1001, + } + mount_text = "1 0 0:1 / / rw,relatime shared:1 - rootfs rootfs rw\n" + mount_snapshot = { + "bytes": len(mount_text.encode("ascii")), + "content": mount_text, + "content_encoding": "ascii", + "content_sha256": hashlib.sha256(mount_text.encode("ascii")).hexdigest(), + "gid": 1001, + "kind": "file", + "mode": 0o444, + "path": "/proc/self/mountinfo", + "status": "present_readable", + "uid": 1001, + } + present_zero = { + "bytes": 2, + "content": "0\n", + "content_encoding": "ascii", + "content_sha256": hashlib.sha256(b"0\n").hexdigest(), + "gid": 0, + "kind": "file", + "mode": 0o444, + "path": HOST_POLICY_CONTROL_PATHS[0], + "status": "present_readable", + "uid": 0, + } + present_false = { + **present_zero, + "bytes": 2, + "content": "N\n", + "content_sha256": hashlib.sha256(b"N\n").hexdigest(), + "path": HOST_POLICY_CONTROL_PATHS[4], + } + absent = {"path": HOST_POLICY_CONTROL_PATHS[1], "status": "absent"} + unreadable = { + "error": "Permission denied", + "error_type": "PermissionError", + "errno": 13, + "operation": "read", + "path": HOST_POLICY_CONTROL_PATHS[2], + "status": "unreadable", + } + directory_entries = ["apparmor", "lsm"] + directory = { + "entries": directory_entries, + "entries_sha256": hashlib.sha256(bytes_for(directory_entries)).hexdigest(), + "gid": 0, + "kind": "directory", + "mode": 0o555, + "path": HOST_POLICY_CONTROL_PATHS[3], + "status": "present_readable", + "uid": 0, + } + controls = {} + for path in HOST_POLICY_CONTROL_PATHS: + controls[path] = {"path": path, "status": "absent"} + controls[HOST_POLICY_CONTROL_PATHS[0]] = present_zero + controls[HOST_POLICY_CONTROL_PATHS[1]] = absent + controls[HOST_POLICY_CONTROL_PATHS[2]] = unreadable + controls[HOST_POLICY_CONTROL_PATHS[3]] = directory + controls[HOST_POLICY_CONTROL_PATHS[4]] = present_false + synthetic = { + "captured_state": valid, + "controls": controls, + "format": "dsv41-host-policy-diagnostic", + "proc_self_status": { + "fields": state_fields, + "snapshot": status_snapshot, + }, + "root_mount_propagation": { + "root_mounts": root_mount_evidence(mount_snapshot), + "snapshot": mount_snapshot, + }, + "uname": { + "machine": "aarch64", "nodename": "runner", "release": "6.11", + "status": "available", "sysname": "Linux", "version": "#1", + }, + "version": 1, + } + validate_host_policy_diagnostic(synthetic, valid) + unavailable_status = { + **synthetic, + "proc_self_status": { + "fields": { + name: {"status": "unavailable"} + for name in HOST_POLICY_STATUS_FIELDS + }, + "snapshot": {"path": "/proc/self/status", "status": "absent"}, + }, + "uname": { + "error": "unavailable", "error_type": "OSError", + "errno": None, "status": "unavailable", + }, + } + validate_host_policy_diagnostic(unavailable_status, valid) + partial_status = { + **synthetic, + "proc_self_status": { + "fields": { + **state_fields, + "Seccomp_filters": {"status": "absent"}, + }, + "snapshot": status_snapshot, + }, + } + validate_host_policy_diagnostic(partial_status, valid) + unavailable_mount = { + **synthetic, + "root_mount_propagation": { + "root_mounts": [], + "snapshot": {"path": "/proc/self/mountinfo", "status": "absent"}, + }, + } + validate_host_policy_diagnostic(unavailable_mount, valid) + for mutation in ( + {"format": "wrong"}, + {"version": 2}, + {"captured_state": {**valid, "no_new_privs": 0}}, + {"controls": {**controls, "/wrong": {"path": "/wrong", "status": "absent"}}}, + {"proc_self_status": { + "fields": {**state_fields, "NoNewPrivs": {"raw": "0", "status": "present"}}, + "snapshot": status_snapshot, + }}, + {"root_mount_propagation": {"root_mounts": [], "snapshot": mount_snapshot}}): + changed = dict(synthetic) + changed.update(mutation) + rejected(lambda changed=changed: validate_host_policy_diagnostic(changed, valid)) + for invalid, expected_path in ( + ({**present_zero, "content": "1\n"}, HOST_POLICY_CONTROL_PATHS[0]), + ({**present_zero, "bytes": 1}, HOST_POLICY_CONTROL_PATHS[0]), + ({**present_zero, "content_sha256": "0" * 64}, HOST_POLICY_CONTROL_PATHS[0]), + ({**absent, "content": ""}, HOST_POLICY_CONTROL_PATHS[1]), + ({**unreadable, "error": ""}, HOST_POLICY_CONTROL_PATHS[2]), + ({**unreadable, "content": ""}, HOST_POLICY_CONTROL_PATHS[2]), + ({**directory, "entries_sha256": "0" * 64}, HOST_POLICY_CONTROL_PATHS[3]), + ({"path": HOST_POLICY_CONTROL_PATHS[0], "status": "invalid"}, HOST_POLICY_CONTROL_PATHS[0]), + ({**present_zero, "path": "/wrong"}, HOST_POLICY_CONTROL_PATHS[0])): + rejected(lambda invalid=invalid, expected_path=expected_path: validate_snapshot(invalid, expected_path)) + root = Path(output).resolve().parent / "gate-self-test" + root.mkdir(mode=0o700) + with tempfile.TemporaryDirectory(dir=root) as temp: + workspace = Path(temp) / "workspace" + build = workspace / "build" + direct = build / "direct" + direct.mkdir(parents=True, mode=0o700) + workspace.chmod(0o700) + build.chmod(0o700) + directory_receipts(workspace, build, [direct], os.getuid()) + direct.chmod(0o770) + rejected(lambda: directory_receipts(workspace, build, [direct], os.getuid())) + direct.chmod(0o702) + rejected(lambda: directory_receipts(workspace, build, [direct], os.getuid())) + direct.chmod(0o700) + link = build / "link" + link.symlink_to(direct, target_is_directory=True) + rejected(lambda: directory_receipts(workspace, build, [link], os.getuid())) + marker = build / "exclusive.json" + write_exclusive(marker, {"value": 1}) + rejected(lambda: write_exclusive(marker, {"value": 2})) + digest = hashlib.sha256(marker.read_bytes()).hexdigest() + marker.chmod(0o600) + marker.write_text('{"value":2}\n', encoding="ascii") + rejected(lambda: verify_digest(marker, digest)) + Path(output).write_text(json.dumps({ + "environment": "PASS", "exclusive_and_tamper": "PASS", + "host_policy_diagnostic": "PASS", + "interpreter": "PASS", "secure_ancestry": "PASS", + "state_matrix": "PASS", + }, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + + if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--self-test": + self_test(sys.argv[2]) + elif sys.argv[1:] == ["--launch"]: + launch() + else: + raise SystemExit("invalid fixture gate arguments") + PY + cat > "$ci_root/hook/sitecustomize.py" <<'PY' + import hashlib + import importlib.util + import json + import os + import sys + import time + import unittest + from pathlib import Path + + def load_gate(): + path = Path(os.environ["DSV41_CI_GATE"]).resolve(strict=True) + if hashlib.sha256(path.read_bytes()).hexdigest() != os.environ["DSV41_CI_GATE_SHA256"]: + raise RuntimeError("fixture gate SHA-256 mismatch") + spec = importlib.util.spec_from_file_location("dsv41_ci_fixture_gate", path) + if spec is None or spec.loader is None: + raise RuntimeError("cannot load fixture gate") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + GATE = load_gate() + CONFIG = GATE.load_config() + HOOK_PATH = Path(__file__).resolve(strict=True) + HOOK_SHA256 = GATE.verify_digest(HOOK_PATH, os.environ["DSV41_CI_HOOK_SHA256"]) + SOURCE = Path(CONFIG["source"]).resolve(strict=True) + MARKER_DIR = Path(CONFIG["paths"]["marker"]).resolve(strict=True) + TEST_CLASS = "TraceFormatTests" + TEST_METHOD = "test_native_watchdog_validation_crosses_private_pid_namespace" + TEST_ID = f"__main__.{TEST_CLASS}.{TEST_METHOD}" + BINDINGS = ( + "DSV41_NATIVE_TRACE_BINARY", "DSV41_NATIVE_CONTAINMENT_HELPER", + "DSV41_NATIVE_MANIFEST_BINARY", "DSV41_NATIVE_INJECT_LIBRARY", + ) + ORIGINAL_CALL = unittest.TestCase._callTestMethod + ORIGINAL_SKIP = unittest.TextTestResult.addSkip + + def preflight(): + path = Path(os.environ["DSV41_CI_PREFLIGHT"]).resolve(strict=True) + digest = GATE.verify_digest(path, os.environ["DSV41_CI_PREFLIGHT_SHA256"]) + return path, digest, json.loads(path.read_text(encoding="ascii")) + + def live_evidence(): + path, digest, before = preflight() + state = GATE.current_state() + GATE.validate_state(state, CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) + diagnostic = GATE.validate_host_policy_file( + before["host_policy_diagnostic"], state, + Path(CONFIG["paths"]["evidence"]) / "host-policy-diagnostic.json") + interpreter = GATE.validate_interpreter(CONFIG) + if any(os.environ.get(name) for name in GATE.FORBIDDEN): + raise RuntimeError("target process has a forbidden loader environment") + paths = CONFIG["paths"] + direct = [paths[name] for name in ("evidence", "hook", "log", "marker", "tmp")] + receipts = GATE.directory_receipts(paths["workspace"], paths["build"], direct, CONFIG["runner_uid"]) + if state != before["state"] or interpreter != before["interpreter"] or receipts != before["secure_directories"]: + raise RuntimeError("target process differs from preflight") + return { + "active_forbidden_loader_environment": [], + "host_policy_diagnostic": diagnostic, + "interpreter": interpreter, + "preflight_path": str(path), + "preflight_sha256": digest, + "secure_directories": receipts, + "state": state, + } + + def call_test_method(self, method): + function = getattr(method, "__func__", method) + code = getattr(function, "__code__", None) + source = Path(code.co_filename).resolve(strict=True) if code is not None else None + if self.__class__.__name__ == TEST_CLASS and method.__name__ == TEST_METHOD and source == SOURCE: + bindings = {} + for name in BINDINGS: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"missing {name} binding") + bindings[name] = str(Path(value).resolve(strict=True)) + payload = { + **live_evidence(), "bindings": bindings, "event": "test_method_entry", + "first_line": code.co_firstlineno, "function": TEST_METHOD, + "gate_sha256": os.environ["DSV41_CI_GATE_SHA256"], + "hook_sha256": HOOK_SHA256, + "pid": os.getpid(), "ppid": os.getppid(), "source": str(source), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "test_id": self.id(), "timestamp_monotonic_ns": time.monotonic_ns(), + } + try: + GATE.write_exclusive(MARKER_DIR / "entry.json", payload) + except FileExistsError: + GATE.write_exclusive( + MARKER_DIR / f"duplicate-{os.getpid()}-{time.monotonic_ns()}.json", payload) + raise RuntimeError(f"{TEST_ID} entered more than once") + return ORIGINAL_CALL(self, method) + + def add_skip(self, test, reason): + method = getattr(test, getattr(test, "_testMethodName", ""), None) + function = getattr(method, "__func__", method) + code = getattr(function, "__code__", None) + source = Path(code.co_filename).resolve(strict=True) if code is not None else None + test_id = test.id() + _path, digest, _before = preflight() + payload = { + "event": "test_skip", "preflight_sha256": digest, "reason": reason, + "source": str(source) if source is not None else None, + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest() if source is not None else None, + "test_id": test_id, + } + GATE.write_exclusive( + MARKER_DIR / f"skip-{hashlib.sha256(test_id.encode('ascii')).hexdigest()}.json", payload) + return ORIGINAL_SKIP(self, test, reason) + + unittest.TestCase._callTestMethod = call_test_method + unittest.TextTestResult.addSkip = add_skip + PY + cat > "$ci_root/evidence/postgate.py" <<'PY' + import ast + import hashlib + import importlib.util + import json + import os + import re + import sys + import tempfile + from pathlib import Path + from xml.etree import ElementTree + + def load_gate(): + path = Path(os.environ["DSV41_CI_GATE"]).resolve(strict=True) + if hashlib.sha256(path.read_bytes()).hexdigest() != os.environ["DSV41_CI_GATE_SHA256"]: + raise RuntimeError("fixture gate SHA-256 mismatch") + spec = importlib.util.spec_from_file_location("dsv41_ci_fixture_gate_post", path) + if spec is None or spec.loader is None: + raise RuntimeError("cannot load fixture gate") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + GATE = load_gate() + CONFIG = GATE.load_config() + TARGET_METHOD = "test_native_watchdog_validation_crosses_private_pid_namespace" + TARGET_ID = f"__main__.TraceFormatTests.{TARGET_METHOD}" + APPROVED_SKIP_ID = "__main__.TraceFormatTests.test_unproven_posix_containment_fails_closed_before_setsid_escape" + BINDING_NAMES = { + "DSV41_NATIVE_TRACE_BINARY", "DSV41_NATIVE_CONTAINMENT_HELPER", + "DSV41_NATIVE_MANIFEST_BINARY", "DSV41_NATIVE_INJECT_LIBRARY", + } + + def junit_values(path): + root = ElementTree.parse(path).getroot() + return {name: int(root.attrib.get(name, -1)) for name in ("tests", "failures", "disabled", "skipped")} + + def validate_outer(status, junit, log): + if status != 0: + raise RuntimeError(f"CTest failed with status {status}") + if junit != {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}: + raise RuntimeError(f"CTest JUnit result is invalid: {junit}") + if "***Skipped" in log or "***Not Run" in log: + raise RuntimeError("CTest was skipped or not run") + start_pattern = re.compile(r"^\s*Start\s+[1-9]\d*:\s+test-deepseek41-trace\s*$") + starts = sum(start_pattern.fullmatch(line) is not None for line in log.splitlines()) + if starts != 1: + raise RuntimeError(f"registered CTest execution count is invalid: {starts}") + + def expected_bindings(metadata): + tests = metadata.get("tests", []) + if len(tests) != 1 or tests[0].get("name") != "test-deepseek41-trace": + raise RuntimeError("CTest metadata selector is invalid") + properties = {item["name"]: item["value"] for item in tests[0].get("properties", [])} + environment = properties.get("ENVIRONMENT", []) + if isinstance(environment, str): + environment = environment.split(";") + result = {} + for value in environment: + name, setting = value.split("=", 1) + if name.startswith("DSV41_NATIVE_"): + result[name] = str(Path(setting).resolve(strict=True)) + if set(result) != BINDING_NAMES: + raise RuntimeError("native bindings are invalid") + return result + + def target_first_line(source): + tree = ast.parse(source.read_text(encoding="ascii")) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "TraceFormatTests": + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)) and member.name == TARGET_METHOD: + return min([member.lineno, *(decorator.lineno for decorator in member.decorator_list)]) + raise RuntimeError("target method is absent") + + def validate_first_line(actual, expected): + if actual != expected: + raise RuntimeError(f"target first line differs: actual={actual} expected={expected}") + + def validate_counts(entries, duplicates, target_skips, total_skips): + if entries != 1 or duplicates != 0 or target_skips != 0 or total_skips != 1: + raise RuntimeError( + f"marker counts are invalid: entry={entries} duplicate={duplicates} " + f"target_skip={target_skips} skips={total_skips}") + + def validate_preflight_diagnostic(paths): + evidence = Path(paths["evidence"]) + preflight_path = evidence / "runner-preflight.json" + preflight_sha = hashlib.sha256(preflight_path.read_bytes()).hexdigest() + preflight = json.loads(preflight_path.read_text(encoding="ascii")) + GATE.validate_state( + preflight["state"], CONFIG["runner_uid"], CONFIG["runner_gid"], CONFIG["runner_cap_bnd"]) + diagnostic = GATE.validate_host_policy_file( + preflight["host_policy_diagnostic"], preflight["state"], + evidence / "host-policy-diagnostic.json") + return diagnostic, preflight_path, preflight_sha, preflight + + def validate_marker(status, paths, preflight_evidence=None): + evidence = Path(paths["evidence"]) + marker_dir = Path(paths["marker"]) + log = (Path(paths["log"]) / "test-deepseek41-trace-ctest.log").read_text(errors="replace") + diagnostic, preflight_path, preflight_sha, preflight = ( + validate_preflight_diagnostic(paths) + if preflight_evidence is None else preflight_evidence) + validate_outer( + status, + junit_values(evidence / "test-deepseek41-trace-ctest.xml"), + log, + ) + metadata = json.loads((evidence / "test-deepseek41-trace-ctest-metadata.json").read_text()) + bindings = expected_bindings(metadata) + if preflight["environment_names"] != CONFIG["environment_allowlist"]: + raise RuntimeError("preflight environment allowlist differs") + if preflight["ctest"] != str(Path(CONFIG["ctest"]).resolve(strict=True)): + raise RuntimeError("preflight CTest identity differs") + if preflight["interpreter"] != GATE.validate_interpreter(CONFIG): + raise RuntimeError("preflight Python identity differs") + direct = [paths[name] for name in ("evidence", "hook", "log", "marker", "tmp")] + receipts = GATE.directory_receipts( + paths["workspace"], paths["build"], direct, CONFIG["runner_uid"]) + if preflight["secure_directories"] != receipts: + raise RuntimeError("preflight secure directory evidence differs") + entries = list(marker_dir.glob("entry.json")) + duplicates = list(marker_dir.glob("duplicate-*.json")) + skips = list(marker_dir.glob("skip-*.json")) + unexpected = [ + path for path in marker_dir.iterdir() + if path not in entries and path not in duplicates and path not in skips + ] + skip_payloads = [json.loads(path.read_text(encoding="ascii")) for path in skips] + target_skips = sum(payload.get("test_id") == TARGET_ID for payload in skip_payloads) + validate_counts(len(entries), len(duplicates), target_skips, len(skips)) + if unexpected: + raise RuntimeError(f"unexpected marker artifacts: {unexpected}") + marker = json.loads(entries[0].read_text(encoding="ascii")) + source = Path(CONFIG["source"]).resolve(strict=True) + expected = { + "active_forbidden_loader_environment": [], + "bindings": bindings, + "event": "test_method_entry", + "function": TARGET_METHOD, + "gate_sha256": os.environ["DSV41_CI_GATE_SHA256"], + "host_policy_diagnostic": diagnostic, + "hook_sha256": os.environ["DSV41_CI_HOOK_SHA256"], + "interpreter": preflight["interpreter"], + "preflight_path": str(preflight_path), + "preflight_sha256": preflight_sha, + "secure_directories": receipts, + "source": str(source), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "state": preflight["state"], + "test_id": TARGET_ID, + } + for name, value in expected.items(): + if marker.get(name) != value: + raise RuntimeError(f"target entry marker {name} differs") + validate_first_line(marker.get("first_line"), target_first_line(source)) + if not isinstance(marker.get("pid"), int) or marker["pid"] <= 1: + raise RuntimeError("target entry marker PID is invalid") + if not isinstance(marker.get("ppid"), int) or marker["ppid"] <= 0: + raise RuntimeError("target entry marker PPID is invalid") + if not isinstance(marker.get("timestamp_monotonic_ns"), int) or marker["timestamp_monotonic_ns"] <= 0: + raise RuntimeError("target entry marker timestamp is invalid") + approved = { + "event": "test_skip", + "preflight_sha256": preflight_sha, + "reason": "Linux uses subreaper and pidfd containment", + "source": str(source), + "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "test_id": APPROVED_SKIP_ID, + } + if skip_payloads != [approved]: + raise RuntimeError("approved skip identity differs") + return { + "ctest_execution_count": 1, + "duplicate_entry_count": 0, + "forbidden_loader_environment": [], + "host_policy_diagnostic": diagnostic, + "junit": {"disabled": 0, "failures": 0, "skipped": 0, "tests": 1}, + "preflight_sha256": preflight_sha, + "result": "PASS", + "target_entry_count": 1, + "target_skip_count": 0, + } + + def rejected(function): + try: + function() + except Exception: + return + raise RuntimeError("negative postgate fixture was accepted") + + def self_test(output): + validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n") + rejected(lambda: validate_outer( + 1, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 1, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 1}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 1, "skipped": 0}, + " Start 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + " Start 35: test-deepseek41-trace\n***Not Run\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, "")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + "Start 35: test-deepseek41-trace\nStart 35: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + "Start 0: test-deepseek41-trace\n")) + rejected(lambda: validate_outer( + 0, {"tests": 1, "failures": 0, "disabled": 0, "skipped": 0}, + "Start 35: wrong-test\n")) + decorated = ast.parse( + "class TraceFormatTests:\n" + " @staticmethod\n" + " def test_native_watchdog_validation_crosses_private_pid_namespace():\n" + " pass\n") + member = decorated.body[0].body[0] + first_line = min([member.lineno, *(item.lineno for item in member.decorator_list)]) + validate_first_line(first_line, 2) + rejected(lambda: validate_first_line(member.lineno, 2)) + validate_counts(1, 0, 0, 1) + for counts in ((0, 0, 0, 1), (1, 1, 0, 1), (1, 0, 1, 1), (1, 0, 0, 2)): + rejected(lambda counts=counts: validate_counts(*counts)) + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp) / "marker.json" + GATE.write_exclusive(marker, {"preflight_sha256": "a"}) + rejected(lambda: GATE.write_exclusive(marker, {"preflight_sha256": "b"})) + rejected(lambda: GATE.verify_digest(marker, "0" * 64)) + Path(output).write_text(json.dumps({ + "duplicate_and_race": "PASS", "junit_failure_skip_notrun": "PASS", + "preflight_spoof": "PASS", "target_nonentry_skip_duplicate": "PASS", + }, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + + def validate(status, output): + preflight_evidence = None + try: + preflight_evidence = validate_preflight_diagnostic(CONFIG["paths"]) + receipt = validate_marker(status, CONFIG["paths"], preflight_evidence) + except Exception as error: + receipt = {"error": str(error), "result": "FAIL"} + if preflight_evidence is not None: + receipt["host_policy_diagnostic"] = preflight_evidence[0] + GATE.write_exclusive(Path(output), receipt) + raise + GATE.write_exclusive(Path(output), receipt) + + if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--self-test": + self_test(sys.argv[2]) + elif len(sys.argv) == 4 and sys.argv[1] == "--validate": + validate(int(sys.argv[2]), sys.argv[3]) + else: + raise SystemExit("invalid postgate arguments") + PY + chmod 0444 \ + "$ci_root/evidence/fixture-config.json" \ + "$ci_root/evidence/fixture_gate.py" \ + "$ci_root/evidence/forbidden-loader-environment.json" \ + "$ci_root/evidence/postgate.py" \ + "$ci_root/hook/sitecustomize.py" + export DSV41_CI_CONFIG="$ci_root/evidence/fixture-config.json" + export DSV41_CI_CONFIG_SHA256="$(sha256sum "$DSV41_CI_CONFIG" | cut -d' ' -f1)" + export DSV41_CI_GATE="$ci_root/evidence/fixture_gate.py" + export DSV41_CI_GATE_SHA256="$(sha256sum "$DSV41_CI_GATE" | cut -d' ' -f1)" + export DSV41_CI_HOOK_SHA256="$(sha256sum "$ci_root/hook/sitecustomize.py" | cut -d' ' -f1)" + printf '%s %s\n' "$DSV41_CI_GATE_SHA256" "$DSV41_CI_GATE" > "$ci_root/evidence/fixture_gate.sha256" + printf '%s %s\n' "$DSV41_CI_HOOK_SHA256" "$ci_root/hook/sitecustomize.py" > "$ci_root/evidence/sitecustomize.sha256" + "$selected_interpreter" "$ci_root/evidence/fixture_gate.py" \ + --self-test "$ci_root/evidence/fixture-gate-self-test.json" + "$selected_interpreter" "$ci_root/evidence/postgate.py" \ + --self-test "$ci_root/evidence/postgate-self-test.json" + : > "$ci_root/log/test-deepseek41-trace-ctest.log" + set +e + sudo -n -- /usr/bin/setpriv \ + --reuid="$runner_uid" \ + --regid="$runner_gid" \ + --clear-groups \ + --inh-caps=-all \ + --ambient-caps=-all \ + --no-new-privs \ + -- /usr/bin/env -i \ + "PATH=${selected_interpreter%/*}:/usr/bin:/bin" \ + "HOME=$HOME" \ + "LANG=C" \ + "LC_ALL=C" \ + "TMPDIR=$ci_root/tmp" \ + "PYTHONDONTWRITEBYTECODE=1" \ + "PYTHONCOERCECLOCALE=0" \ + "PYTHONPATH=$ci_root/hook" \ + "PYTHONUTF8=1" \ + "DSV41_CI_CONFIG=$DSV41_CI_CONFIG" \ + "DSV41_CI_CONFIG_SHA256=$DSV41_CI_CONFIG_SHA256" \ + "DSV41_CI_GATE=$DSV41_CI_GATE" \ + "DSV41_CI_GATE_SHA256=$DSV41_CI_GATE_SHA256" \ + "DSV41_CI_HOOK_SHA256=$DSV41_CI_HOOK_SHA256" \ + "$selected_interpreter" "$ci_root/evidence/fixture_gate.py" --launch \ + 2>&1 | tee "$ci_root/log/test-deepseek41-trace-ctest.log" + ctest_status="${PIPESTATUS[0]}" + set -e + "$selected_interpreter" "$ci_root/evidence/postgate.py" \ + --validate "$ctest_status" "$ci_root/evidence/postgate-result.json" + + - name: Upload DeepSeek V4.1 trace CTest evidence + if: ${{ always() }} + uses: actions/upload-artifact@v6 with: - python-version: '3.11' - pip-install: -r tools/server/tests/requirements.txt + name: deepseek-v41-trace-ctest-${{ github.run_id }}-${{ github.run_attempt }} + path: | + build/deepseek-v41-trace-ci/ + if-no-files-found: error + retention-days: 7 - name: Tests id: server_integration_tests diff --git a/README.md b/README.md index 3067a7a4e71e..dbf547ce5c5b 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Everything else is upstream `llama.cpp`. The additions currently carried here: | Speculative checkpoints on device | | `llama-server` keeps speculative-decoding checkpoints in device memory instead of copying them to the host | | ROCmFPx quant types | `llama-quantize` types `Q4_0_ROCMFP4`, `Q4_0_ROCMFP4_FAST`, `Q2/Q3/Q6/Q8_0_ROCMFPX` and the `_LEAN`/`_COHERENT`/`_STRIX` recipes | Loads the ROCmFP4 GGUFs published for Strix Halo. CPU codecs plus Vulkan dequant, mat-vec, matmul and integer-dot kernels. Weight formats only: not accepted as KV-cache types | | Repeatable output at depth | | Freed KV cells are zeroed so masked-out rows never carry stale K/V, and the Vulkan radix top-k assigns output slots deterministically | +| Host-memory watchdog | [`scripts/strix_memory_watchdog.py`](docs/strix-memory-watchdog.md) | Runs a command in a process group, requires zero active swap, and stops before host-wide memory reaches the 120 GiB validation ceiling | Every ROCm/HIP change above is guarded on architecture, shape and layout, so other devices see upstream behaviour. Run `--help`, or see [tools/server/README.md](tools/server/README.md), for the full options. diff --git a/cmake/build-info.cmake b/cmake/build-info.cmake index c7005950c561..edf89ebb1458 100644 --- a/cmake/build-info.cmake +++ b/cmake/build-info.cmake @@ -18,7 +18,7 @@ endif() # Get the commit count and hash if(Git_FOUND) execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + COMMAND ${GIT_EXECUTABLE} rev-parse HEAD WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE HEAD OUTPUT_STRIP_TRAILING_WHITESPACE diff --git a/docs/strix-memory-watchdog.md b/docs/strix-memory-watchdog.md new file mode 100644 index 000000000000..f0099b9a55ad --- /dev/null +++ b/docs/strix-memory-watchdog.md @@ -0,0 +1,94 @@ +# Strix host-memory watchdog + +`scripts/strix_memory_watchdog.py` is an external Linux command wrapper for headless Strix Halo validation. It does not change model loading or cache sizing. It measures host-wide memory from procfs and controls the launched command's process group. + +```sh +./scripts/strix_memory_watchdog.py -- ./build/bin/llama-server +``` + +The wrapper performs these checks and actions: + +- It refuses to launch if `/proc/swaps` contains any active entry. +- It calculates used memory as `MemTotal - MemAvailable`. Linux reports these fields in KiB, so the wrapper multiplies each value by 1024 and keeps all accounting as integer bytes. +- It sends `SIGTERM` to the process group at 116 GiB used. +- It sends `SIGKILL` at 118 GiB used or 30 seconds after `SIGTERM`. +- It reports `grace_timeout` if any descendant requires `SIGKILL` after the soft-threshold grace period, even when the direct child exited earlier. +- It sends `SIGKILL` and fails if swap appears or required procfs data becomes unavailable during execution. +- It forwards wrapper `SIGHUP`, `SIGINT`, or `SIGTERM` to the process group, waits the configured grace period, then sends `SIGKILL` if any group member remains. +- It checks the process group after the direct child exits and cleans up remaining descendants before returning the child's classification. +- It applies the same bounded process-group cleanup if an unexpected post-launch error occurs. +- It propagates an unmonitored child exit code. A signal exit uses the shell convention `128 + signal`. + +The 118 GiB emergency threshold leaves a 2 GiB sampling margin below the strict 120 GiB ceiling. The default sample interval is one second. This margin cannot guarantee the ceiling for a workload that can allocate more than 2 GiB between samples. Lower `--emergency-gib` or shorten `--sample-interval-seconds` for such a workload. + +Use `--procfs-root` to select a different procfs mount or a test fixture. `--soft-gib`, `--emergency-gib`, `--grace-seconds`, and `--sample-interval-seconds` override the other defaults. The emergency threshold must remain below 120 GiB. The fail-closed timing bounds are a maximum 30-second grace, maximum one-second sample interval, and maximum five-second heartbeat age. + +The wrapper writes timestamped JSON Lines records to standard error. Preflight, sample, signal, and final records include total, available, used, and peak-used bytes, swap entry count, child status, process-group status, threshold reason, and final classification where applicable. Signal records are written immediately after each process-group signal. Child standard input, standard output, and standard error are inherited unchanged. + +## Watchdog-owned validation lease + +Use all three artifact options together when another process must prove that it is inside the active watchdog process group: + +```sh +./scripts/strix_memory_watchdog.py \ + --lease-path /run/deepseek-v41/watchdog-lease.json \ + --heartbeat-path /run/deepseek-v41/watchdog-heartbeat.json \ + --audit-path /run/deepseek-v41/watchdog-audit.jsonl \ + -- \ + python3 tools/deepseek-v41-trace/run_matrix.py +``` + +The watchdog creates and exclusively locks the persistent audit before launch. It then starts an internal guardian as the new session and process-group leader; the guardian starts the supplied command in that same group without inheriting the private control pipe. After the guardian reports the payload PID, the watchdog atomically creates the lease and heartbeat. Existing artifact paths are rejected rather than overwritten. The payload receives the resolved paths through `STRIX_MEMORY_WATCHDOG_LEASE_PATH`, `STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH`, and `STRIX_MEMORY_WATCHDOG_AUDIT_PATH`. It also receives `STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS`. + +The child can run before the first atomic lease rename. A matching preflight must retry the inherited lease path for a bounded interval and fail closed if a complete valid lease does not appear. It must not accept a lease path supplied separately by the operator. Consumers must require version 2; version 1 does not describe the guardian topology or timing policy and is rejected. + +Lease format `strix-memory-watchdog-lease`, version 2, contains: + +- `lease_id` and active/final `state` +- `watchdog_pid`, `watchdog_start_time_utc`, Linux `watchdog_start_time_ticks`, `watchdog_executable_path`, `watchdog_command_sha256`, `watchdog_script_path`, and `watchdog_script_sha256` +- exact `soft_bytes`, `emergency_bytes`, `strict_ceiling_bytes`, `grace_seconds`, and `sample_interval_seconds` +- `procfs_root` +- `guardian_pid`, payload `child_pid`, `child_process_group_id`, `command`, and `child_command_sha256` +- `heartbeat_path`, `max_heartbeat_age_seconds`, and `audit_path` +- device, inode, owner, and mode identity for atomic JSON artifacts, plus the watchdog-held audit descriptor identity +- the authoritative `final` audit record after termination + +Heartbeat format `strix-memory-watchdog-heartbeat`, version 2, binds `lease_id`, watchdog PID/start ticks, child PID/process group, sequence, state, and update timestamps. Every memory sample first checks swap and memory thresholds, pulses the guardian through the private nonblocking pipe, then atomically replaces the heartbeat with the complete sample audit record and its persistent-audit record hash. It pulses again after persistence succeeds. A blocked audit or heartbeat write cannot delay the emergency signal; if persistence stalls past the guardian deadline, the guardian fails closed. A final heartbeat and final lease update remain on disk with the persistent JSONL audit; the watchdog does not delete this evidence. + +The guardian uses Linux `PR_SET_PDEATHSIG` with a parent-race check. It kills its process group on watchdog death, control-pipe EOF/error, or a missed pulse deadline, including a stopped or wedged watchdog. When the watchdog sends a graceful signal, it also puts the guardian into a bounded grace mode and continues private pulses while it waits. This lets the watchdog own the configured grace deadline and record any `SIGKILL` escalation instead of letting the shorter heartbeat deadline preempt cleanup. If the grace control message or a cleanup pulse fails, the watchdog independently sends `SIGKILL` to the process group and reaps the child before it reports `signal_error`. The payload must call `start_process_group_lease_guard()` before it starts exporter descendants. This validates the lease with bounded startup retries, arms a second parent-death link to the guardian, and starts a thread that kills the process group if any validation or artifact operation fails or the watchdog evidence becomes stale. + +A matching Linux preflight must verify all of the following: + +- The inherited lease, heartbeat, and audit paths match the paths inside the lease. +- `/proc//exe` is the exact expected Python executable and argv position 1 is the exact repository watchdog script. `-c`, `-m`, helper-script, inert-argument, and interpreter-option substitutions are rejected. +- The watchdog command line itself supplies the exact 116/118 GiB thresholds, `/proc`, inherited artifact paths, timing policy, and command after `--`; the lease cannot override those expectations. +- `/proc//stat` start ticks and `/proc//cmdline` SHA-256 match the lease and remain stable across validation. A pidfd is held during validation when Linux provides `pidfd_open`. +- The topology is watchdog parent -> guardian process-group leader -> payload child. The current process must be inside `child_process_group_id`. +- The command identity is expected, the procfs root is `/proc`, and thresholds are exactly 116 GiB soft, 118 GiB emergency, and 120 GiB strict ceiling for the final run. +- Lease and heartbeat files are regular, mode 0600, owned by the current UID, opened with `O_NOFOLLOW`, and match their recorded device/inode identity. +- The heartbeat identity matches the lease, its monotonic timestamp is not older than `max_heartbeat_age_seconds`, and its audit-record hash exists in the persistent audit. +- The persistent audit matches the watchdog-held descriptor device/inode and remains exclusively locked by the live watchdog. + +These checks reject accidental or helper-process substitution and make regular-file heartbeat forgery unable to keep the process group alive after private pulses stop. They are not a security boundary against intentionally hostile code running as the same UID; use a separately owned systemd user service or cgroup if that threat is in scope. + +The guardian controls only the process group. A payload that deliberately calls `setsid()` can escape it. The correctness harness must not do that. If arbitrary payload code is in scope, launch the watchdog in a service/cgroup configured to kill every member when the unit stops. + +Exit classifications are authoritative in the last final JSON record. If final artifact persistence fails after a primary safety failure, the primary classification and exit code remain unchanged and the artifact failure is listed in `secondary_errors`. Operational failures use these exit codes: + +| Exit code | Classification | +| ---: | --- | +| 2 | procfs or configuration error | +| 3 | swap active at startup or detected during execution | +| 4 | soft threshold reached | +| 5 | emergency threshold reached | +| 6 | soft-threshold grace period expired | +| 7 | process-group signaling or termination failure | +| 8 | lease, heartbeat, or persistent audit failure | +| 70 | unexpected post-launch error | +| 127 | command launch failure | + +No model, backend, or ROCm package is required to run the unit tests: + +```sh +python3 tests/test_strix_memory_watchdog.py +``` diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index ba9bc83b9b08..c022dc0d5a09 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -13,7 +13,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH) if(GIT_EXE) # Get current git commit hash - execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD + execute_process(COMMAND ${GIT_EXE} rev-parse HEAD WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GGML_BUILD_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE diff --git a/scripts/strix_memory_watchdog.py b/scripts/strix_memory_watchdog.py new file mode 100755 index 000000000000..a06c85de3f96 --- /dev/null +++ b/scripts/strix_memory_watchdog.py @@ -0,0 +1,2575 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import ctypes +import fcntl +import hashlib +import json +import math +import os +import re +import secrets +import select +import signal +import stat +import subprocess +import sys +import threading +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import IO, Any, Protocol + + +GIB = 1024**3 +STRICT_CEILING_BYTES = 120 * GIB +DEFAULT_SOFT_BYTES = 116 * GIB +DEFAULT_EMERGENCY_BYTES = 118 * GIB +DEFAULT_GRACE_SECONDS = 30.0 +DEFAULT_SAMPLE_INTERVAL_SECONDS = 1.0 +DEFAULT_HEARTBEAT_MAX_AGE_SECONDS = 5.0 +MAX_GRACE_SECONDS = 30.0 +MAX_SAMPLE_INTERVAL_SECONDS = 1.0 +MAX_HEARTBEAT_MAX_AGE_SECONDS = 5.0 + +LEASE_FORMAT = "strix-memory-watchdog-lease" +LEASE_VERSION = 2 +HEARTBEAT_FORMAT = "strix-memory-watchdog-heartbeat" +HEARTBEAT_VERSION = 2 +PR_SET_PDEATHSIG = 1 +LEASE_GUARD_SIGNAL = signal.SIGUSR1 + +EXIT_PROCFS_ERROR = 2 +EXIT_SWAP_ACTIVE = 3 +EXIT_SOFT_LIMIT = 4 +EXIT_EMERGENCY_LIMIT = 5 +EXIT_GRACE_TIMEOUT = 6 +EXIT_SIGNAL_ERROR = 7 +EXIT_LEASE_ERROR = 8 +EXIT_INTERNAL_ERROR = 70 +EXIT_LAUNCH_ERROR = 127 + +MEMINFO_VALUE_RE = re.compile(r"([0-9]+) kB") +SWAPS_HEADER = ["Filename", "Type", "Size", "Used", "Priority"] +PARENT_SIGNALS = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM) + + +class ProcfsError(RuntimeError): + pass + + +class ProcessGroupError(RuntimeError): + pass + + +class ArtifactError(RuntimeError): + def __init__(self, component: str, detail: str): + self.component = component + super().__init__(detail) + + +class LeaseValidationError(RuntimeError): + pass + + +class ParentSignal(RuntimeError): + def __init__(self, signal_number: int): + self.signal_number = signal_number + super().__init__(signal.Signals(signal_number).name) + + +class ProcessHandle(Protocol): + pid: int + + def poll(self) -> int | None: + ... + + def wait(self, timeout: float | None = None) -> int: + ... + + +@dataclass +class GuardianProcess: + process: subprocess.Popen[bytes] + payload_pid: int + pulse_fd: int + + @property + def pid(self) -> int: + return self.process.pid + + def poll(self) -> int | None: + return self.process.poll() + + def wait(self, timeout: float | None = None) -> int: + return self.process.wait(timeout=timeout) + + def pulse(self) -> None: + self._write_control(b"P") + + def begin_grace(self) -> None: + self._write_control(b"G") + + def _write_control(self, value: bytes) -> None: + try: + os.write(self.pulse_fd, value) + except BlockingIOError as exc: + raise ProcessGroupError( + "guardian pulse pipe is blocked" + ) from exc + except OSError as exc: + detail = exc.strerror or str(exc) + raise ProcessGroupError( + f"cannot pulse guardian: {detail}" + ) from exc + + def close(self) -> None: + try: + os.close(self.pulse_fd) + except OSError: + pass + + +@dataclass(frozen=True) +class HostSnapshot: + total_bytes: int + available_bytes: int + active_swaps: tuple[str, ...] + + @property + def used_bytes(self) -> int: + return self.total_bytes - self.available_bytes + + +@dataclass +class RuntimeState: + snapshot: HostSnapshot + peak_used_bytes: int + + +@dataclass(frozen=True) +class ArtifactPaths: + lease: Path + heartbeat: Path + audit: Path + + +@dataclass(frozen=True) +class WatchdogConfig: + command: tuple[str, ...] + procfs_root: Path = Path("/proc") + soft_bytes: int = DEFAULT_SOFT_BYTES + emergency_bytes: int = DEFAULT_EMERGENCY_BYTES + grace_seconds: float = DEFAULT_GRACE_SECONDS + sample_interval_seconds: float = DEFAULT_SAMPLE_INTERVAL_SECONDS + lease_path: Path | None = None + heartbeat_path: Path | None = None + audit_path: Path | None = None + heartbeat_max_age_seconds: float = DEFAULT_HEARTBEAT_MAX_AGE_SECONDS + + @property + def lease_enabled(self) -> bool: + return self.lease_path is not None + + def validate(self) -> ArtifactPaths | None: + if not self.command: + raise ValueError("a command is required after --") + if self.soft_bytes <= 0: + raise ValueError("soft threshold must be greater than zero") + if self.emergency_bytes <= self.soft_bytes: + raise ValueError("emergency threshold must be greater than soft threshold") + if self.emergency_bytes >= STRICT_CEILING_BYTES: + raise ValueError("emergency threshold must be below 120 GiB") + if ( + not math.isfinite(self.grace_seconds) + or self.grace_seconds <= 0 + or self.grace_seconds > MAX_GRACE_SECONDS + ): + raise ValueError( + "grace period must be greater than zero and at most 30 seconds" + ) + if ( + not math.isfinite(self.sample_interval_seconds) + or self.sample_interval_seconds <= 0 + or self.sample_interval_seconds > MAX_SAMPLE_INTERVAL_SECONDS + ): + raise ValueError( + "sample interval must be greater than zero and at most 1 second" + ) + if ( + not math.isfinite(self.heartbeat_max_age_seconds) + or self.heartbeat_max_age_seconds + <= self.sample_interval_seconds + or self.heartbeat_max_age_seconds + > MAX_HEARTBEAT_MAX_AGE_SECONDS + ): + raise ValueError( + "heartbeat max age must be greater than sample interval " + "and at most 5 seconds" + ) + lease_paths = ( + self.lease_path, + self.heartbeat_path, + self.audit_path, + ) + if any(path is not None for path in lease_paths) and not all( + path is not None for path in lease_paths + ): + raise ValueError( + "lease, heartbeat, and audit paths must be specified together" + ) + if self.lease_enabled: + assert self.lease_path is not None + assert self.heartbeat_path is not None + assert self.audit_path is not None + try: + paths = ArtifactPaths( + self.lease_path.expanduser().resolve(), + self.heartbeat_path.expanduser().resolve(), + self.audit_path.expanduser().resolve(), + ) + except (OSError, RuntimeError) as exc: + raise ValueError( + f"cannot resolve watchdog artifact path: {exc}" + ) from exc + if len({paths.lease, paths.heartbeat, paths.audit}) != 3: + raise ValueError( + "lease, heartbeat, and audit paths must be distinct" + ) + return paths + return None + + +class ProcfsReader: + def __init__(self, root: Path): + self.root = root + + def _read_text(self, name: str) -> str: + path = self.root / name + try: + return path.read_text(encoding="utf-8") + except OSError as exc: + detail = exc.strerror or str(exc) + raise ProcfsError(f"cannot read {path}: {detail}") from exc + + def read_snapshot(self) -> HostSnapshot: + active_swaps = self._parse_swaps(self._read_text("swaps")) + total_bytes, available_bytes = self._parse_meminfo( + self._read_text("meminfo") + ) + return HostSnapshot(total_bytes, available_bytes, active_swaps) + + @staticmethod + def _parse_meminfo(content: str) -> tuple[int, int]: + values: dict[str, int] = {} + required = {"MemTotal", "MemAvailable"} + for line in content.splitlines(): + key, separator, raw_value = line.partition(":") + if not separator or key not in required: + continue + if key in values: + raise ProcfsError(f"duplicate {key} in meminfo") + match = MEMINFO_VALUE_RE.fullmatch(raw_value.strip()) + if match is None: + raise ProcfsError(f"malformed {key} in meminfo") + values[key] = int(match.group(1)) * 1024 + + missing = sorted(required - values.keys()) + if missing: + raise ProcfsError(f"missing {', '.join(missing)} in meminfo") + if values["MemAvailable"] > values["MemTotal"]: + raise ProcfsError("MemAvailable exceeds MemTotal") + return values["MemTotal"], values["MemAvailable"] + + @staticmethod + def _parse_swaps(content: str) -> tuple[str, ...]: + lines = content.splitlines() + if not lines or lines[0].split() != SWAPS_HEADER: + raise ProcfsError("malformed swaps header") + + entries: list[str] = [] + for line in lines[1:]: + if not line.strip(): + continue + fields = line.split() + if len(fields) != len(SWAPS_HEADER): + raise ProcfsError("malformed swaps entry") + try: + int(fields[2]) + int(fields[3]) + int(fields[4]) + except ValueError as exc: + raise ProcfsError("malformed swaps entry") from exc + entries.append(fields[0]) + return tuple(entries) + + +def _timestamp_utc( + wall_clock: Callable[[], datetime] | None = None, +) -> str: + timestamp = (wall_clock or ( + lambda: datetime.now(timezone.utc) + ))().astimezone(timezone.utc) + return timestamp.isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ) + + +def _sha256_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def _sha256_file(path: Path) -> str: + try: + return _sha256_bytes(path.read_bytes()) + except OSError as exc: + detail = exc.strerror or str(exc) + raise ArtifactError( + "lease", f"cannot hash {path}: {detail}" + ) from exc + + +def _command_sha256(command: Sequence[str]) -> str: + encoded = json.dumps( + list(command), + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + return _sha256_bytes(encoded) + + +def _set_parent_death_signal( + signal_number: int, expected_parent_pid: int +) -> None: + if not sys.platform.startswith("linux"): + return + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(PR_SET_PDEATHSIG, signal_number, 0, 0, 0) != 0: + error_number = ctypes.get_errno() + raise OSError(error_number, os.strerror(error_number)) + if os.getppid() != expected_parent_pid: + os.kill(os.getpid(), signal.SIGKILL) + + +def _kill_own_process_group( + _signal_number: int | None = None, + _frame: object | None = None, +) -> None: + try: + os.killpg(os.getpgrp(), signal.SIGKILL) + except OSError: + os._exit(EXIT_SIGNAL_ERROR) + + +def _guardian_main( + control_fd: int, + status_fd: int, + pulse_timeout_seconds: float, + grace_timeout_seconds: float, + command: tuple[str, ...], +) -> int: + if not sys.platform.startswith("linux"): + return EXIT_LAUNCH_ERROR + os.set_inheritable(control_fd, False) + os.set_inheritable(status_fd, False) + signal.signal(LEASE_GUARD_SIGNAL, _kill_own_process_group) + for signal_number in PARENT_SIGNALS: + signal.signal(signal_number, signal.SIG_IGN) + _set_parent_death_signal(LEASE_GUARD_SIGNAL, os.getppid()) + + def prepare_payload() -> None: + for signal_number in PARENT_SIGNALS: + signal.signal(signal_number, signal.SIG_DFL) + + try: + payload = subprocess.Popen(command, preexec_fn=prepare_payload) + except (OSError, ValueError) as exc: + os.write( + status_fd, + json.dumps( + {"error": getattr(exc, "strerror", None) or str(exc)} + ).encode("utf-8") + + b"\n", + ) + os.close(status_fd) + return EXIT_LAUNCH_ERROR + + os.write( + status_fd, + json.dumps({"payload_pid": payload.pid}).encode("utf-8") + b"\n", + ) + os.close(status_fd) + poller = select.poll() + poller.register( + control_fd, + select.POLLIN | select.POLLHUP | select.POLLERR, + ) + current_timeout_seconds = pulse_timeout_seconds + deadline = time.monotonic() + pulse_timeout_seconds + while True: + remaining = max(0.0, deadline - time.monotonic()) + events = poller.poll(max(1, min(50, int(remaining * 1000)))) + for _, event_mask in events: + if event_mask & (select.POLLHUP | select.POLLERR): + _kill_own_process_group() + try: + pulse = os.read(control_fd, 65536) + except BlockingIOError: + pulse = b"" + if not pulse: + _kill_own_process_group() + if b"G" in pulse: + current_timeout_seconds = grace_timeout_seconds + deadline = time.monotonic() + current_timeout_seconds + if time.monotonic() >= deadline: + _kill_own_process_group() + returncode = payload.poll() + if returncode is not None: + if returncode >= 0: + return returncode + signal_number = -returncode + if signal_number not in (signal.SIGKILL, signal.SIGSTOP): + signal.signal(signal_number, signal.SIG_DFL) + os.kill(os.getpid(), signal_number) + return 128 + signal_number + + +def _read_guardian_status( + descriptor: int, timeout_seconds: float +) -> int: + poller = select.poll() + poller.register(descriptor, select.POLLIN | select.POLLHUP) + deadline = time.monotonic() + timeout_seconds + content = b"" + while time.monotonic() < deadline: + events = poller.poll( + max(1, int((deadline - time.monotonic()) * 1000)) + ) + if not events: + continue + chunk = os.read(descriptor, 4096) + if not chunk: + break + content += chunk + if b"\n" in content: + break + if not content: + raise OSError("guardian did not report payload startup") + try: + status = json.loads(content.splitlines()[0]) + except (UnicodeError, json.JSONDecodeError) as exc: + raise OSError("guardian returned malformed startup status") from exc + if not isinstance(status, dict): + raise OSError("guardian returned malformed startup status") + if "error" in status: + raise OSError(str(status["error"])) + payload_pid = status.get("payload_pid") + if not isinstance(payload_pid, int): + raise OSError("guardian did not report a payload PID") + return payload_pid + + +def _launch_guardian( + command: tuple[str, ...], + environment: dict[str, str], + pulse_timeout_seconds: float, + grace_timeout_seconds: float, + launch_mask: set[signal.Signals], +) -> GuardianProcess: + control_read, control_write = os.pipe() + os.set_blocking(control_read, False) + os.set_blocking(control_write, False) + status_read, status_write = os.pipe() + parent_pid = os.getpid() + + def prepare_guardian() -> None: + signal.pthread_sigmask(signal.SIG_SETMASK, launch_mask) + _set_parent_death_signal(signal.SIGKILL, parent_pid) + + guardian_command = ( + sys.executable, + str(Path(__file__).resolve()), + "--internal-guardian", + str(control_read), + str(status_write), + str(pulse_timeout_seconds), + str(grace_timeout_seconds), + "--", + *command, + ) + try: + process = subprocess.Popen( + guardian_command, + start_new_session=True, + pass_fds=(control_read, status_write), + preexec_fn=prepare_guardian, + env=environment, + ) + finally: + os.close(control_read) + os.close(status_write) + try: + payload_pid = _read_guardian_status(status_read, 5.0) + except OSError: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5.0) + os.close(control_write) + raise + finally: + os.close(status_read) + guardian = GuardianProcess(process, payload_pid, control_write) + guardian.pulse() + return guardian + + +def _read_proc_bytes(root: Path, process_id: int, name: str) -> bytes: + path = root / str(process_id) / name + try: + return path.read_bytes() + except OSError as exc: + detail = exc.strerror or str(exc) + raise LeaseValidationError( + f"cannot read {path}: {detail}" + ) from exc + + +def _parse_proc_stat(content: str) -> tuple[int, int, int]: + close_paren = content.rfind(")") + if close_paren < 0: + raise LeaseValidationError("malformed process stat") + fields = content[close_paren + 1:].split() + if len(fields) < 20: + raise LeaseValidationError("malformed process stat") + try: + return int(fields[1]), int(fields[2]), int(fields[19]) + except ValueError as exc: + raise LeaseValidationError("malformed process stat") from exc + + +def _read_proc_stat( + root: Path, process_id: int +) -> tuple[int, int, int]: + content = _read_proc_bytes( + root, process_id, "stat" + ).decode("utf-8") + return _parse_proc_stat(content) + + +def _write_json_atomic( + path: Path, + value: dict[str, object], + *, + create: bool = False, +) -> None: + parent = path.parent + temp_path = parent / ( + f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp" + ) + try: + descriptor = os.open( + temp_path, + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o600, + ) + with os.fdopen(descriptor, "wb") as stream: + file_status = os.fstat(stream.fileno()) + record = { + **value, + "file_device": file_status.st_dev, + "file_inode": file_status.st_ino, + "file_uid": file_status.st_uid, + "file_mode": stat.S_IMODE(file_status.st_mode), + } + payload = ( + json.dumps( + record, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode("utf-8") + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + if create: + os.link(temp_path, path) + temp_path.unlink() + else: + os.replace(temp_path, path) + directory_descriptor = os.open(parent, os.O_RDONLY) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + except OSError as exc: + try: + temp_path.unlink() + except FileNotFoundError: + pass + detail = exc.strerror or str(exc) + action = "create" if create else "write" + raise ArtifactError( + "lease", f"cannot atomically {action} {path}: {detail}" + ) from exc + + +class LeaseManager: + def __init__( + self, + config: WatchdogConfig, + paths: ArtifactPaths, + *, + process_procfs_root: Path = Path("/proc"), + wall_clock: Callable[[], datetime] | None = None, + monotonic_ns: Callable[[], int] | None = None, + ): + self.config = config + self.lease_path = paths.lease + self.heartbeat_path = paths.heartbeat + self.audit_path = paths.audit + self.process_procfs_root = process_procfs_root + self.wall_clock = wall_clock + self.monotonic_ns = monotonic_ns or time.monotonic_ns + self.lease_id = secrets.token_hex(16) + self.sequence = 0 + self.lease: dict[str, object] | None = None + + def _watchdog_identity(self) -> dict[str, object]: + script_path = Path(__file__).resolve() + cmdline_path = ( + self.process_procfs_root / str(os.getpid()) / "cmdline" + ) + proc_start_time_ticks: int | None = None + try: + cmdline = cmdline_path.read_bytes() + _, _, proc_start_time_ticks = _read_proc_stat( + self.process_procfs_root, os.getpid() + ) + executable_path = ( + self.process_procfs_root + / str(os.getpid()) + / "exe" + ).resolve() + except (OSError, LeaseValidationError): + if sys.platform.startswith("linux"): + raise ArtifactError( + "lease", + "cannot read watchdog process identity from procfs", + ) + cmdline = b"\0".join( + os.fsencode(argument) for argument in sys.argv + ) + executable_path = Path(sys.executable).resolve() + return { + "pid": os.getpid(), + "start_time_utc": _timestamp_utc(self.wall_clock), + "proc_start_time_ticks": proc_start_time_ticks, + "cmdline_sha256": _sha256_bytes(cmdline), + "executable_path": str(executable_path), + "script_path": str(script_path), + "script_sha256": _sha256_file(script_path), + } + + def _heartbeat_record( + self, + state: str, + sample: dict[str, object] | None = None, + ) -> dict[str, object]: + assert self.lease is not None + self.sequence += 1 + record: dict[str, object] = { + "format": HEARTBEAT_FORMAT, + "version": HEARTBEAT_VERSION, + "lease_id": self.lease_id, + "sequence": self.sequence, + "state": state, + "updated_at": _timestamp_utc(self.wall_clock), + "updated_monotonic_ns": self.monotonic_ns(), + "watchdog_pid": self.lease["watchdog_pid"], + "watchdog_start_time_ticks": ( + self.lease["watchdog_start_time_ticks"] + ), + "child_pid": self.lease["child_pid"], + "child_process_group_id": ( + self.lease["child_process_group_id"] + ), + } + if sample is not None: + record["sample"] = sample + return record + + def start( + self, child: ProcessHandle, audit: AuditLogger + ) -> None: + watchdog_identity = self._watchdog_identity() + audit_identity = audit.persistent_identity() + payload_pid = ( + child.payload_pid + if isinstance(child, GuardianProcess) + else child.pid + ) + self.lease = { + "format": LEASE_FORMAT, + "version": LEASE_VERSION, + "lease_id": self.lease_id, + "state": "active", + "watchdog_pid": watchdog_identity["pid"], + "watchdog_start_time_utc": ( + watchdog_identity["start_time_utc"] + ), + "watchdog_start_time_ticks": ( + watchdog_identity["proc_start_time_ticks"] + ), + "watchdog_command_sha256": ( + watchdog_identity["cmdline_sha256"] + ), + "watchdog_executable_path": ( + watchdog_identity["executable_path"] + ), + "watchdog_script_path": watchdog_identity["script_path"], + "watchdog_script_sha256": ( + watchdog_identity["script_sha256"] + ), + "soft_bytes": self.config.soft_bytes, + "emergency_bytes": self.config.emergency_bytes, + "strict_ceiling_bytes": STRICT_CEILING_BYTES, + "grace_seconds": self.config.grace_seconds, + "sample_interval_seconds": self.config.sample_interval_seconds, + "guardian_pid": child.pid, + "child_pid": payload_pid, + "child_process_group_id": child.pid, + "command": list(self.config.command), + "child_command_sha256": _command_sha256( + self.config.command + ), + "heartbeat_path": str(self.heartbeat_path), + "max_heartbeat_age_seconds": ( + self.config.heartbeat_max_age_seconds + ), + "audit_path": str(self.audit_path), + "audit_device": audit_identity["device"], + "audit_inode": audit_identity["inode"], + "audit_uid": audit_identity["uid"], + "audit_mode": audit_identity["mode"], + "audit_fd": audit_identity["fd"], + "procfs_root": str( + self.config.procfs_root.expanduser().resolve() + ), + } + heartbeat = self._heartbeat_record( + "active", + {"audit_record_sha256": audit.last_record_sha256}, + ) + _write_json_atomic(self.heartbeat_path, heartbeat, create=True) + _write_json_atomic(self.lease_path, self.lease, create=True) + + def update_heartbeat(self, sample: dict[str, object]) -> None: + heartbeat = self._heartbeat_record("active", sample) + _write_json_atomic(self.heartbeat_path, heartbeat) + + def finalize(self, final_record: dict[str, object]) -> None: + if self.lease is None: + return + self.lease["state"] = "final" + self.lease["final"] = final_record + heartbeat = self._heartbeat_record("final") + _write_json_atomic(self.heartbeat_path, heartbeat) + _write_json_atomic(self.lease_path, self.lease) + + +def _read_json_object(path: Path) -> dict[str, object]: + try: + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + with os.fdopen(descriptor, "r", encoding="utf-8") as stream: + file_status = os.fstat(stream.fileno()) + if ( + not stat.S_ISREG(file_status.st_mode) + or file_status.st_uid != os.getuid() + or stat.S_IMODE(file_status.st_mode) != 0o600 + ): + raise LeaseValidationError( + f"{path} has unsafe type, owner, or mode" + ) + value = json.load(stream) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise LeaseValidationError( + f"cannot read valid JSON from {path}: {exc}" + ) from exc + if not isinstance(value, dict): + raise LeaseValidationError(f"{path} must contain a JSON object") + if ( + value.get("file_device") != file_status.st_dev + or value.get("file_inode") != file_status.st_ino + or value.get("file_uid") != file_status.st_uid + or value.get("file_mode") != stat.S_IMODE(file_status.st_mode) + ): + raise LeaseValidationError(f"{path} identity does not match") + return value + + +def _require_int(value: object, field: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise LeaseValidationError(f"lease field {field} is invalid") + return value + + +def _require_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value: + raise LeaseValidationError(f"lease field {field} is invalid") + return value + + +def validate_active_lease( + lease_path: Path, + *, + expected_script_path: Path, + expected_executable_path: Path | None = None, + expected_soft_bytes: int = DEFAULT_SOFT_BYTES, + expected_emergency_bytes: int = DEFAULT_EMERGENCY_BYTES, + expected_procfs_root: Path = Path("/proc"), + expected_command: Sequence[str] | None = None, + expected_heartbeat_path: Path | None = None, + expected_audit_path: Path | None = None, + expected_max_heartbeat_age_seconds: float | None = None, + current_process_id: int | None = None, + process_procfs_root: Path = Path("/proc"), + monotonic_ns: Callable[[], int] | None = None, + pidfd_open: Callable[[int], int] | None = getattr( + os, "pidfd_open", None + ), +) -> dict[str, object]: + lease_path = lease_path.expanduser().resolve() + lease = _read_json_object(lease_path) + if ( + lease.get("format") != LEASE_FORMAT + or lease.get("version") != LEASE_VERSION + or lease.get("state") != "active" + ): + raise LeaseValidationError("lease format, version, or state is invalid") + + script_path = Path( + _require_string( + lease.get("watchdog_script_path"), + "watchdog_script_path", + ) + ).resolve() + expected_script_path = expected_script_path.expanduser().resolve() + if script_path != expected_script_path: + raise LeaseValidationError("watchdog script path does not match") + script_sha256 = _require_string( + lease.get("watchdog_script_sha256"), + "watchdog_script_sha256", + ) + if script_sha256 != _sha256_file(expected_script_path): + raise LeaseValidationError("watchdog script SHA does not match") + + if ( + _require_int( + lease.get("soft_bytes"), "soft_bytes" + ) + != expected_soft_bytes + or _require_int( + lease.get("emergency_bytes"), + "emergency_bytes", + ) + != expected_emergency_bytes + or _require_int( + lease.get("strict_ceiling_bytes"), + "strict_ceiling_bytes", + ) + != STRICT_CEILING_BYTES + ): + raise LeaseValidationError("watchdog thresholds do not match") + lease_procfs_root = Path( + _require_string(lease.get("procfs_root"), "procfs_root") + ).resolve() + if lease_procfs_root != expected_procfs_root.expanduser().resolve(): + raise LeaseValidationError("watchdog procfs root does not match") + + watchdog_pid = _require_int( + lease.get("watchdog_pid"), "watchdog_pid" + ) + pidfd: int | None = None + if pidfd_open is not None: + try: + pidfd = pidfd_open(watchdog_pid) + except OSError as exc: + raise LeaseValidationError( + "cannot open watchdog pidfd" + ) from exc + watchdog_start_ticks = _require_int( + lease.get("watchdog_start_time_ticks"), + "watchdog_start_time_ticks", + ) + _, _, live_watchdog_start_ticks = _read_proc_stat( + process_procfs_root, watchdog_pid + ) + if live_watchdog_start_ticks != watchdog_start_ticks: + raise LeaseValidationError("watchdog process start time does not match") + expected_executable = ( + expected_executable_path or Path(sys.executable) + ).expanduser().resolve() + try: + live_executable = ( + process_procfs_root / str(watchdog_pid) / "exe" + ).resolve() + except OSError as exc: + raise LeaseValidationError( + "cannot resolve watchdog executable" + ) from exc + if ( + live_executable != expected_executable + or Path( + _require_string( + lease.get("watchdog_executable_path"), + "watchdog_executable_path", + ) + ).resolve() + != expected_executable + ): + raise LeaseValidationError("watchdog executable does not match") + live_cmdline = _read_proc_bytes( + process_procfs_root, watchdog_pid, "cmdline" + ) + if _sha256_bytes(live_cmdline) != _require_string( + lease.get("watchdog_command_sha256"), + "watchdog_command_sha256", + ): + raise LeaseValidationError("watchdog command line does not match") + argv = [ + os.fsdecode(argument) + for argument in live_cmdline.split(b"\0") + if argument + ] + if len(argv) < 2 or argv[1] in ("-c", "-m"): + raise LeaseValidationError( + "watchdog script is not in executable argv position" + ) + try: + watchdog_cwd = ( + process_procfs_root / str(watchdog_pid) / "cwd" + ).resolve() + except OSError as exc: + raise LeaseValidationError( + "cannot resolve watchdog working directory" + ) from exc + argv_script = Path(argv[1]).expanduser() + if not argv_script.is_absolute(): + argv_script = watchdog_cwd / argv_script + if argv_script.resolve() != expected_script_path: + raise LeaseValidationError( + "watchdog script is not in executable argv position" + ) + try: + live_config = parse_args(argv[2:]) + live_paths = live_config.validate() + except (SystemExit, ValueError) as exc: + raise LeaseValidationError( + "watchdog command line is invalid" + ) from exc + if ( + live_config.soft_bytes != expected_soft_bytes + or live_config.emergency_bytes != expected_emergency_bytes + or live_config.procfs_root.expanduser().resolve() + != expected_procfs_root.expanduser().resolve() + ): + raise LeaseValidationError( + "watchdog command-line policy does not match" + ) + if ( + lease.get("grace_seconds") != live_config.grace_seconds + or lease.get("sample_interval_seconds") + != live_config.sample_interval_seconds + or lease.get("max_heartbeat_age_seconds") + != live_config.heartbeat_max_age_seconds + ): + raise LeaseValidationError( + "watchdog lease timing policy does not match" + ) + if ( + live_paths is None + or live_paths.lease != lease_path + or ( + expected_heartbeat_path is not None + and live_paths.heartbeat + != expected_heartbeat_path.expanduser().resolve() + ) + or ( + expected_audit_path is not None + and live_paths.audit + != expected_audit_path.expanduser().resolve() + ) + ): + raise LeaseValidationError( + "watchdog command-line artifact paths do not match" + ) + if expected_command is not None and tuple( + expected_command + ) != live_config.command: + raise LeaseValidationError("monitored command does not match") + + guardian_pid = _require_int( + lease.get("guardian_pid"), "guardian_pid" + ) + child_pid = _require_int(lease.get("child_pid"), "child_pid") + process_group_id = _require_int( + lease.get("child_process_group_id"), + "child_process_group_id", + ) + guardian_parent_pid, guardian_group_id, _ = _read_proc_stat( + process_procfs_root, guardian_pid + ) + child_parent_pid, child_group_id, _ = _read_proc_stat( + process_procfs_root, child_pid + ) + if ( + guardian_parent_pid != watchdog_pid + or guardian_group_id != process_group_id + or guardian_pid != process_group_id + or child_parent_pid != guardian_pid + or child_group_id != process_group_id + ): + raise LeaseValidationError( + "watchdog, guardian, child, or process group does not match" + ) + command = lease.get("command") + if ( + not isinstance(command, list) + or not command + or not all(isinstance(argument, str) for argument in command) + ): + raise LeaseValidationError("lease field command is invalid") + command_sha256 = _require_string( + lease.get("child_command_sha256"), + "child_command_sha256", + ) + if command_sha256 != _command_sha256(command): + raise LeaseValidationError("monitored command SHA is invalid") + if expected_command is not None and command_sha256 != _command_sha256( + expected_command + ): + raise LeaseValidationError("monitored command SHA does not match") + + process_id = ( + current_process_id + if current_process_id is not None + else os.getpid() + ) + _, current_group_id, _ = _read_proc_stat( + process_procfs_root, process_id + ) + if current_group_id != process_group_id: + raise LeaseValidationError( + "current process is outside the monitored process group" + ) + + heartbeat_path = Path( + _require_string( + lease.get("heartbeat_path"), "heartbeat_path" + ) + ).resolve() + if ( + expected_heartbeat_path is not None + and heartbeat_path + != expected_heartbeat_path.expanduser().resolve() + ): + raise LeaseValidationError("heartbeat path does not match") + heartbeat_max_age = lease.get("max_heartbeat_age_seconds") + if ( + not isinstance(heartbeat_max_age, (int, float)) + or isinstance(heartbeat_max_age, bool) + or not math.isfinite(heartbeat_max_age) + or heartbeat_max_age <= 0 + ): + raise LeaseValidationError( + "lease field max_heartbeat_age_seconds is invalid" + ) + if ( + expected_max_heartbeat_age_seconds is not None + and heartbeat_max_age != expected_max_heartbeat_age_seconds + ): + raise LeaseValidationError("heartbeat max age does not match") + heartbeat = _read_json_object(heartbeat_path) + lease_id = _require_string(lease.get("lease_id"), "lease_id") + if ( + heartbeat.get("format") != HEARTBEAT_FORMAT + or heartbeat.get("version") != HEARTBEAT_VERSION + or heartbeat.get("state") != "active" + or heartbeat.get("lease_id") != lease_id + or heartbeat.get("watchdog_pid") != watchdog_pid + or heartbeat.get("watchdog_start_time_ticks") + != watchdog_start_ticks + or heartbeat.get("child_pid") != child_pid + or heartbeat.get("child_process_group_id") != process_group_id + ): + raise LeaseValidationError("heartbeat identity does not match lease") + updated_monotonic_ns = _require_int( + heartbeat.get("updated_monotonic_ns"), + "heartbeat.updated_monotonic_ns", + ) + _require_int(heartbeat.get("sequence"), "heartbeat.sequence") + _require_string(heartbeat.get("updated_at"), "heartbeat.updated_at") + heartbeat_sample = heartbeat.get("sample") + if not isinstance(heartbeat_sample, dict): + raise LeaseValidationError("heartbeat sample is invalid") + audit_record_sha256 = _require_string( + heartbeat_sample.get("audit_record_sha256"), + "heartbeat.sample.audit_record_sha256", + ) + now_monotonic_ns = (monotonic_ns or time.monotonic_ns)() + age_ns = now_monotonic_ns - updated_monotonic_ns + if age_ns < 0 or age_ns > int(heartbeat_max_age * 1_000_000_000): + raise LeaseValidationError("watchdog heartbeat is stale") + + audit_path = Path( + _require_string(lease.get("audit_path"), "audit_path") + ).resolve() + if ( + expected_audit_path is not None + and audit_path != expected_audit_path.expanduser().resolve() + ): + raise LeaseValidationError("persistent audit path does not match") + audit_fd = _require_int(lease.get("audit_fd"), "audit_fd") + audit_device = _require_int( + lease.get("audit_device"), "audit_device" + ) + audit_inode = _require_int( + lease.get("audit_inode"), "audit_inode" + ) + audit_uid = _require_int(lease.get("audit_uid"), "audit_uid") + audit_mode = _require_int(lease.get("audit_mode"), "audit_mode") + try: + audit_status = audit_path.stat(follow_symlinks=False) + live_audit_status = ( + process_procfs_root + / str(watchdog_pid) + / "fd" + / str(audit_fd) + ).stat() + if ( + not stat.S_ISREG(audit_status.st_mode) + or audit_status.st_dev != audit_device + or audit_status.st_ino != audit_inode + or live_audit_status.st_dev != audit_device + or live_audit_status.st_ino != audit_inode + or audit_status.st_uid != audit_uid + or audit_uid != os.getuid() + or stat.S_IMODE(audit_status.st_mode) != audit_mode + or audit_mode != 0o600 + ): + raise LeaseValidationError( + "persistent audit identity does not match" + ) + audit_descriptor = os.open( + audit_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + try: + fcntl.flock( + audit_descriptor, + fcntl.LOCK_EX | fcntl.LOCK_NB, + ) + except BlockingIOError: + pass + else: + fcntl.flock(audit_descriptor, fcntl.LOCK_UN) + raise LeaseValidationError( + "watchdog does not hold the persistent audit lock" + ) + finally: + os.close(audit_descriptor) + audit_lines = [ + line + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + if line + ] + first_line = next(iter(audit_lines)) + first_record = json.loads(first_line) + if ( + not isinstance(first_record, dict) + or not isinstance(first_record.get("event"), str) + or not isinstance(first_record.get("timestamp"), str) + ): + raise LeaseValidationError( + "persistent audit does not contain watchdog records" + ) + if not any( + _sha256_bytes((line + "\n").encode("utf-8")) + == audit_record_sha256 + for line in audit_lines + ): + raise LeaseValidationError( + "heartbeat audit record does not match persistent audit" + ) + except StopIteration as exc: + raise LeaseValidationError("persistent audit is empty") from exc + except (UnicodeError, json.JSONDecodeError) as exc: + raise LeaseValidationError( + "persistent audit does not contain valid JSONL" + ) from exc + except LeaseValidationError: + raise + except OSError as exc: + raise LeaseValidationError( + f"cannot inspect persistent audit {audit_path}: {exc}" + ) from exc + _, _, final_watchdog_start_ticks = _read_proc_stat( + process_procfs_root, watchdog_pid + ) + if final_watchdog_start_ticks != watchdog_start_ticks: + raise LeaseValidationError( + "watchdog process changed during validation" + ) + if pidfd is not None: + os.close(pidfd) + return lease + + +def start_process_group_lease_guard( + expected_script_path: Path, + *, + startup_timeout_seconds: float = 5.0, + expected_procfs_root: Path = Path("/proc"), + process_procfs_root: Path = Path("/proc"), +) -> threading.Thread: + try: + lease_path = Path( + os.environ["STRIX_MEMORY_WATCHDOG_LEASE_PATH"] + ).resolve() + heartbeat_path = Path( + os.environ["STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH"] + ).resolve() + audit_path = Path( + os.environ["STRIX_MEMORY_WATCHDOG_AUDIT_PATH"] + ).resolve() + max_age_seconds = float( + os.environ[ + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS" + ] + ) + except (KeyError, ValueError) as exc: + raise LeaseValidationError( + "watchdog artifact environment is missing or invalid" + ) from exc + current_cmdline = _read_proc_bytes( + process_procfs_root, os.getpid(), "cmdline" + ) + expected_command = tuple( + os.fsdecode(argument) + for argument in current_cmdline.split(b"\0") + if argument + ) + deadline = time.monotonic() + startup_timeout_seconds + while True: + try: + lease = validate_active_lease( + lease_path, + expected_script_path=expected_script_path, + expected_procfs_root=expected_procfs_root, + expected_command=expected_command, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=max_age_seconds, + process_procfs_root=process_procfs_root, + ) + break + except Exception: + if time.monotonic() >= deadline: + raise + time.sleep(0.01) + + guardian_pid = _require_int( + lease.get("guardian_pid"), "guardian_pid" + ) + signal.signal(LEASE_GUARD_SIGNAL, _kill_own_process_group) + _set_parent_death_signal(LEASE_GUARD_SIGNAL, guardian_pid) + + def monitor() -> None: + interval = min(1.0, max_age_seconds / 3) + while True: + time.sleep(interval) + try: + validate_active_lease( + lease_path, + expected_script_path=expected_script_path, + expected_procfs_root=expected_procfs_root, + expected_command=expected_command, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=max_age_seconds, + process_procfs_root=process_procfs_root, + ) + except Exception: + _kill_own_process_group() + + guard = threading.Thread( + target=monitor, + name="strix-watchdog-lease-guard", + daemon=True, + ) + guard.start() + return guard + + +class AuditLogger: + def __init__( + self, + stream: IO[str], + wall_clock: Callable[[], datetime] | None = None, + ): + self.stream = stream + self.stream_enabled = True + self.wall_clock = wall_clock + self.persistent_stream: IO[str] | None = None + self.lease_manager: LeaseManager | None = None + self.finalized = False + self.final_exit_code = EXIT_INTERNAL_ERROR + self.last_record_sha256: str | None = None + + def open_persistent(self, path: Path) -> None: + resolved_path = path.expanduser().resolve() + try: + descriptor = os.open( + resolved_path, + os.O_CREAT + | os.O_EXCL + | os.O_WRONLY + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + fcntl.flock( + descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB + ) + self.persistent_stream = os.fdopen( + descriptor, "w", encoding="utf-8" + ) + except OSError as exc: + detail = exc.strerror or str(exc) + raise ArtifactError( + "audit", + f"cannot create persistent audit {resolved_path}: {detail}", + ) from exc + + def persistent_identity(self) -> dict[str, int]: + if self.persistent_stream is None: + raise ArtifactError( + "audit", "persistent audit is not open" + ) + file_status = os.fstat(self.persistent_stream.fileno()) + return { + "device": file_status.st_dev, + "inode": file_status.st_ino, + "uid": file_status.st_uid, + "mode": stat.S_IMODE(file_status.st_mode), + "fd": self.persistent_stream.fileno(), + } + + def close(self) -> None: + persistent_stream = self.persistent_stream + self.persistent_stream = None + if persistent_stream is not None: + try: + persistent_stream.close() + except (OSError, ValueError): + pass + + def disable_component(self, component: str) -> None: + if component == "audit": + self.close() + elif component == "lease": + self.lease_manager = None + elif component == "stderr": + self.stream_enabled = False + + def emit(self, event: str, **fields: object) -> dict[str, object]: + record = { + "timestamp": _timestamp_utc(self.wall_clock), + "event": event, + **fields, + } + line = ( + json.dumps(record, sort_keys=True, separators=(",", ":")) + + "\n" + ) + if self.stream_enabled: + try: + self.stream.write(line) + self.stream.flush() + except (OSError, ValueError) as exc: + detail = getattr(exc, "strerror", None) or str(exc) + raise ArtifactError( + "stderr", + f"cannot write standard error audit: {detail}", + ) from exc + self.last_record_sha256 = _sha256_bytes(line.encode("utf-8")) + if self.persistent_stream is not None: + try: + self.persistent_stream.write(line) + self.persistent_stream.flush() + os.fsync(self.persistent_stream.fileno()) + except OSError as exc: + detail = exc.strerror or str(exc) + raise ArtifactError( + "audit", + f"cannot write persistent audit: {detail}", + ) from exc + return record + + def heartbeat(self, sample: dict[str, object]) -> None: + if self.lease_manager is not None: + self.lease_manager.update_heartbeat( + { + **sample, + "audit_record_sha256": self.last_record_sha256, + } + ) + + def finalize(self, record: dict[str, object]) -> None: + if self.lease_manager is not None: + self.lease_manager.finalize(record) + + def mark_final(self, exit_code: int) -> None: + self.finalized = True + self.final_exit_code = exit_code + + +def _child_status(returncode: int | None, started: bool = True) -> str: + if not started: + return "not_started" + if returncode is None: + return "running" + return "signaled" if returncode < 0 else "exited" + + +def _state_fields( + snapshot: HostSnapshot | None, + peak_used_bytes: int | None, + child: ProcessHandle | None, + child_returncode: int | None, + process_group_status: str, + threshold_reason: str, +) -> dict[str, object]: + return { + "total_bytes": snapshot.total_bytes if snapshot else None, + "available_bytes": snapshot.available_bytes if snapshot else None, + "used_bytes": snapshot.used_bytes if snapshot else None, + "swap_entries": len(snapshot.active_swaps) if snapshot else None, + "peak_used_bytes": peak_used_bytes, + "child_pid": child.pid if child else None, + "child_status": _child_status( + child_returncode, started=child is not None + ), + "child_returncode": child_returncode, + "process_group_id": child.pid if child else None, + "process_group_status": process_group_status, + "threshold_reason": threshold_reason, + } + + +def _emit_final( + audit: AuditLogger, + classification: str, + exit_code: int, + reason: str, + snapshot: HostSnapshot | None, + peak_used_bytes: int | None, + child: ProcessHandle | None = None, + child_returncode: int | None = None, + process_group_status: str = "not_created", + error: str | None = None, + preserve_primary_on_artifact_error: bool = False, + secondary_errors: Sequence[dict[str, str]] | None = None, +) -> int: + fields = _state_fields( + snapshot, + peak_used_bytes, + child, + child_returncode, + process_group_status, + reason, + ) + fields.update(classification=classification, exit_code=exit_code) + if error: + fields["error"] = error + if secondary_errors: + fields["secondary_errors"] = list(secondary_errors) + + def record_artifact_error(exc: ArtifactError) -> None: + nonlocal exit_code + detail = { + "component": exc.component, + "detail": str(exc), + } + if preserve_primary_on_artifact_error: + secondary_errors = fields.setdefault( + "secondary_errors", [] + ) + assert isinstance(secondary_errors, list) + secondary_errors.append(detail) + else: + fields.update( + classification="lease_error", + exit_code=EXIT_LEASE_ERROR, + threshold_reason="watchdog artifact finalization failed", + error=f"{exc.component}: {exc}", + ) + exit_code = EXIT_LEASE_ERROR + + def emit_final_record() -> dict[str, object]: + try: + return audit.emit("final", **fields) + except ArtifactError as exc: + audit.disable_component(exc.component) + record_artifact_error(exc) + try: + return audit.emit("final", **fields) + except ArtifactError as exc: + audit.disable_component(exc.component) + record_artifact_error(exc) + return audit.emit("final", **fields) + + previous_mask = signal.pthread_sigmask( + signal.SIG_BLOCK, PARENT_SIGNALS + ) + try: + record = emit_final_record() + try: + audit.finalize(record) + except ArtifactError as exc: + audit.disable_component(exc.component) + record_artifact_error(exc) + emit_final_record() + audit.mark_final(exit_code) + return exit_code + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + + +def _signal_process_group(process_group_id: int, signal_number: int) -> str: + try: + os.killpg(process_group_id, signal_number) + except ProcessLookupError: + return "missing" + except OSError as exc: + name = signal.Signals(signal_number).name + detail = exc.strerror or str(exc) + raise ProcessGroupError( + f"cannot send {name} to process group {process_group_id}: {detail}" + ) from exc + return f"{signal.Signals(signal_number).name.lower()}_sent" + + +def _process_group_alive(process_group_id: int) -> bool: + if sys.platform.startswith("linux"): + try: + process_paths = Path("/proc").iterdir() + for process_path in process_paths: + if not process_path.name.isdigit(): + continue + try: + content = ( + process_path / "stat" + ).read_text(encoding="utf-8") + close_paren = content.rfind(")") + fields = content[close_paren + 1:].split() + if ( + close_paren >= 0 + and len(fields) >= 3 + and fields[0] != "Z" + and int(fields[2]) == process_group_id + ): + return True + except (OSError, UnicodeError, ValueError): + continue + return False + except OSError: + pass + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + detail = exc.strerror or str(exc) + raise ProcessGroupError( + f"cannot inspect process group {process_group_id}: {detail}" + ) from exc + return True + + +def _raise_parent_signal(signal_number: int, _frame: object) -> None: + raise ParentSignal(signal_number) + + +def _set_parent_signal_handlers( + handler: Any, +) -> dict[int, Any]: + previous: dict[int, Any] = {} + for signal_number in PARENT_SIGNALS: + previous[signal_number] = signal.signal(signal_number, handler) + return previous + + +def _restore_parent_signal_handlers( + previous: dict[int, Any], +) -> None: + for signal_number, handler in previous.items(): + signal.signal(signal_number, handler) + + +def _kill_and_finish( + audit: AuditLogger, + child: ProcessHandle, + snapshot: HostSnapshot, + peak_used_bytes: int, + classification: str, + exit_code: int, + reason: str, + signal_group: Callable[[int, int], str], +) -> int: + try: + group_status = signal_group(child.pid, signal.SIGKILL) + except ProcessGroupError as exc: + return _emit_final( + audit, + "signal_error", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "signal_error", + str(exc), + ) + + artifact_error: ArtifactError | None = None + try: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + group_status, + reason, + ), + signal="SIGKILL", + ) + except ArtifactError as exc: + artifact_error = exc + audit.disable_component(exc.component) + try: + child_returncode = child.wait(timeout=5.0) + except subprocess.TimeoutExpired as exc: + return _emit_final( + audit, + "termination_timeout", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "sigkill_timeout", + str(exc), + ) + if artifact_error is not None: + classification = "lease_error" + exit_code = EXIT_LEASE_ERROR + error = f"{artifact_error.component}: {artifact_error}" + else: + error = None + return _emit_final( + audit, + classification, + exit_code, + reason, + snapshot, + peak_used_bytes, + child, + child_returncode, + group_status, + error, + ) + + +def _graceful_cleanup( + audit: AuditLogger, + child: ProcessHandle, + snapshot: HostSnapshot, + peak_used_bytes: int, + classification: str, + exit_code: int, + reason: str, + graceful_signal: int | None, + grace_seconds: float, + signal_group: Callable[[int, int], str], + group_alive: Callable[[int], bool], + monotonic: Callable[[], float], + sleeper: Callable[[float], None], + process_group_status: str = "active", + escalation_result: tuple[str, int, str] | None = None, + error: str | None = None, +) -> int: + escalated = False + artifact_error: ArtifactError | None = None + guardian_control_error: ProcessGroupError | None = None + try: + if graceful_signal is not None: + process_group_status = signal_group( + child.pid, graceful_signal + ) + try: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + process_group_status, + reason, + ), + signal=signal.Signals(graceful_signal).name, + ) + except ArtifactError as exc: + artifact_error = exc + audit.disable_component(exc.component) + if ( + isinstance(child, GuardianProcess) + and child.poll() is None + ): + try: + child.begin_grace() + except ProcessGroupError as exc: + guardian_control_error = exc + deadline = monotonic() + grace_seconds + while ( + guardian_control_error is None + and monotonic() < deadline + ): + child.poll() + if not group_alive(child.pid): + break + if ( + isinstance(child, GuardianProcess) + and child.poll() is None + ): + try: + child.pulse() + except ProcessGroupError as exc: + guardian_control_error = exc + break + sleeper(min(0.05, deadline - monotonic())) + child.poll() + if ( + guardian_control_error is not None + or group_alive(child.pid) + ): + escalated = True + process_group_status = signal_group( + child.pid, signal.SIGKILL + ) + if guardian_control_error is not None: + signal_reason = ( + "guardian control failed during graceful cleanup" + ) + else: + signal_reason = ( + escalation_result[2] + if escalation_result is not None + else reason + ) + try: + audit.emit( + "process_group_signal", + **_state_fields( + snapshot, + peak_used_bytes, + child, + child.poll(), + process_group_status, + signal_reason, + ), + signal="SIGKILL", + ) + except ArtifactError as exc: + if artifact_error is None: + artifact_error = exc + audit.disable_component(exc.component) + except ProcessGroupError as exc: + return _emit_final( + audit, + "signal_error", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "signal_error", + str(exc), + ) + + child_returncode = child.poll() + if child_returncode is None: + try: + child_returncode = child.wait(timeout=5.0) + except subprocess.TimeoutExpired as exc: + return _emit_final( + audit, + "termination_timeout", + EXIT_SIGNAL_ERROR, + reason, + snapshot, + peak_used_bytes, + child, + child.poll(), + "termination_timeout", + str(exc), + ) + + if guardian_control_error is not None: + classification = "signal_error" + exit_code = EXIT_SIGNAL_ERROR + reason = "guardian control failed during graceful cleanup" + error = str(guardian_control_error) + elif escalated and escalation_result is not None: + classification, exit_code, reason = escalation_result + if artifact_error is not None and guardian_control_error is None: + classification = "lease_error" + exit_code = EXIT_LEASE_ERROR + error = f"{artifact_error.component}: {artifact_error}" + return _emit_final( + audit, + classification, + exit_code, + reason, + snapshot, + peak_used_bytes, + child, + child_returncode, + process_group_status, + error, + preserve_primary_on_artifact_error=( + guardian_control_error is not None + ), + secondary_errors=( + [ + { + "component": artifact_error.component, + "detail": str(artifact_error), + } + ] + if ( + guardian_control_error is not None + and artifact_error is not None + ) + else None + ), + ) + + +def _monitor_child( + config: WatchdogConfig, + reader: ProcfsReader, + audit: AuditLogger, + child: ProcessHandle, + state: RuntimeState, + signal_group: Callable[[int, int], str], + group_alive: Callable[[int], bool], + pulse_guardian: Callable[[], None], + monotonic: Callable[[], float], + sleeper: Callable[[float], None], +) -> int: + soft_deadline: float | None = None + + while True: + child_returncode = child.poll() + if child_returncode is not None: + soft_stop = soft_deadline is not None + classification = "soft_limit" if soft_stop else "child_exit" + exit_code = EXIT_SOFT_LIMIT if soft_stop else ( + 128 - child_returncode + if child_returncode < 0 + else child_returncode + ) + reason = ( + "child exited during soft-threshold grace period" + if soft_stop + else "child exited" + ) + if group_alive(child.pid): + grace_seconds = config.grace_seconds + graceful_signal: int | None = signal.SIGTERM + group_status = "active" + if soft_stop: + grace_seconds = max( + 0.0, soft_deadline - monotonic() + ) + graceful_signal = None + group_status = "sigterm_sent" + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + classification, + exit_code, + ( + f"{reason}; process group members still running" + ), + graceful_signal, + grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + group_status, + ( + ( + "grace_timeout", + EXIT_GRACE_TIMEOUT, + "soft-threshold grace period expired with " + "process group members still running", + ) + if soft_stop + else None + ), + ) + return _emit_final( + audit, + classification, + exit_code, + reason, + state.snapshot, + state.peak_used_bytes, + child, + child_returncode, + "leader_exited", + ) + + now = monotonic() + if soft_deadline is not None and now >= soft_deadline: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "grace_timeout", + EXIT_GRACE_TIMEOUT, + "soft-threshold grace period expired", + signal_group, + ) + + try: + state.snapshot = reader.read_snapshot() + except ProcfsError as exc: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "procfs_error", + EXIT_PROCFS_ERROR, + str(exc), + signal_group, + ) + + state.peak_used_bytes = max( + state.peak_used_bytes, state.snapshot.used_bytes + ) + if state.snapshot.active_swaps: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "swap_appeared", + EXIT_SWAP_ACTIVE, + "active swap appeared during execution", + signal_group, + ) + if state.snapshot.used_bytes >= config.emergency_bytes: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "emergency_limit", + EXIT_EMERGENCY_LIMIT, + "used_bytes >= emergency_bytes", + signal_group, + ) + soft_signal_fields: dict[str, object] | None = None + if ( + soft_deadline is None + and state.snapshot.used_bytes >= config.soft_bytes + ): + try: + group_status = signal_group(child.pid, signal.SIGTERM) + except ProcessGroupError as exc: + return _emit_final( + audit, + "signal_error", + EXIT_SIGNAL_ERROR, + "used_bytes >= soft_bytes", + state.snapshot, + state.peak_used_bytes, + child, + child.poll(), + "signal_error", + str(exc), + ) + soft_deadline = now + config.grace_seconds + if isinstance(child, GuardianProcess): + try: + child.begin_grace() + except ProcessGroupError as exc: + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "signal_error", + EXIT_SIGNAL_ERROR, + str(exc), + signal_group, + ) + soft_signal_fields = { + **_state_fields( + state.snapshot, + state.peak_used_bytes, + child, + child.poll(), + group_status, + "used_bytes >= soft_bytes", + ), + "signal": "SIGTERM", + "grace_deadline_monotonic": soft_deadline, + } + try: + pulse_guardian() + except ProcessGroupError as exc: + if child.poll() is not None: + continue + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "signal_error", + EXIT_SIGNAL_ERROR, + str(exc), + signal_group, + ) + if soft_signal_fields is not None: + audit.emit( + "process_group_signal", + **soft_signal_fields, + ) + sample_record = audit.emit( + "sample", + **_state_fields( + state.snapshot, + state.peak_used_bytes, + child, + None, + "active", + "none", + ) + ) + audit.heartbeat(sample_record) + try: + pulse_guardian() + except ProcessGroupError as exc: + if child.poll() is not None: + continue + return _kill_and_finish( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "signal_error", + EXIT_SIGNAL_ERROR, + str(exc), + signal_group, + ) + + sleep_seconds = config.sample_interval_seconds + if soft_deadline is not None: + sleep_seconds = min( + sleep_seconds, + max(0.0, soft_deadline - monotonic()), + ) + sleeper(sleep_seconds) + + +def run_watchdog( + config: WatchdogConfig, + *, + reader: ProcfsReader | None = None, + audit: AuditLogger | None = None, + launcher: Callable[..., ProcessHandle] | None = None, + signal_group: Callable[[int, int], str] | None = None, + group_alive: Callable[[int], bool] | None = None, + monotonic: Callable[[], float] | None = None, + sleeper: Callable[[float], None] | None = None, +) -> int: + artifact_paths = config.validate() + use_guardian = ( + launcher is None and sys.platform.startswith("linux") + ) + reader = reader or ProcfsReader(config.procfs_root) + audit = audit or AuditLogger(sys.stderr) + launcher = launcher or subprocess.Popen + signal_group = signal_group or _signal_process_group + group_alive = group_alive or _process_group_alive + monotonic = monotonic or time.monotonic + sleeper = sleeper or time.sleep + + if artifact_paths is not None: + try: + audit.open_persistent(artifact_paths.audit) + except ArtifactError as exc: + return _emit_final( + audit, + "lease_error", + EXIT_LEASE_ERROR, + "cannot initialize watchdog artifacts", + None, + None, + error=f"{exc.component}: {exc}", + ) + + try: + snapshot = reader.read_snapshot() + except ProcfsError as exc: + return _emit_final( + audit, + "procfs_error", + EXIT_PROCFS_ERROR, + str(exc), + None, + None, + error=str(exc), + ) + + try: + audit.emit( + "preflight", + **_state_fields( + snapshot, + snapshot.used_bytes, + None, + None, + "not_created", + "none", + ), + soft_bytes=config.soft_bytes, + emergency_bytes=config.emergency_bytes, + strict_ceiling_bytes=STRICT_CEILING_BYTES, + ) + except ArtifactError as exc: + audit.disable_component(exc.component) + return _emit_final( + audit, + "lease_error", + EXIT_LEASE_ERROR, + "cannot write watchdog preflight audit", + snapshot, + snapshot.used_bytes, + error=f"{exc.component}: {exc}", + ) + + if snapshot.active_swaps: + return _emit_final( + audit, + "startup_swap_active", + EXIT_SWAP_ACTIVE, + "active swap present before command launch", + snapshot, + snapshot.used_bytes, + ) + if snapshot.used_bytes >= config.emergency_bytes: + return _emit_final( + audit, + "startup_emergency_limit", + EXIT_EMERGENCY_LIMIT, + "used_bytes >= emergency_bytes before launch", + snapshot, + snapshot.used_bytes, + ) + if snapshot.used_bytes >= config.soft_bytes: + return _emit_final( + audit, + "startup_soft_limit", + EXIT_SOFT_LIMIT, + "used_bytes >= soft_bytes before launch", + snapshot, + snapshot.used_bytes, + ) + + previous_mask = signal.pthread_sigmask( + signal.SIG_BLOCK, PARENT_SIGNALS + ) + mask_restored = False + previous_handlers: dict[int, Any] = {} + child: ProcessHandle | None = None + state = RuntimeState(snapshot, snapshot.used_bytes) + try: + launch_mask = previous_mask + lease_manager = ( + LeaseManager(config, artifact_paths) + if artifact_paths is not None + else None + ) + + def restore_child_signal_mask() -> None: + signal.pthread_sigmask(signal.SIG_SETMASK, launch_mask) + + try: + child_environment = os.environ.copy() + if lease_manager is not None: + child_environment.update( + { + "STRIX_MEMORY_WATCHDOG_LEASE_PATH": str( + lease_manager.lease_path + ), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH": str( + lease_manager.heartbeat_path + ), + "STRIX_MEMORY_WATCHDOG_AUDIT_PATH": str( + lease_manager.audit_path + ), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS": ( + str(config.heartbeat_max_age_seconds) + ), + } + ) + if use_guardian: + child = _launch_guardian( + config.command, + child_environment, + config.heartbeat_max_age_seconds, + config.grace_seconds + 1.0, + launch_mask, + ) + elif lease_manager is not None: + child = launcher( + config.command, + start_new_session=True, + preexec_fn=restore_child_signal_mask, + env=child_environment, + ) + else: + child = launcher( + config.command, + start_new_session=True, + preexec_fn=restore_child_signal_mask, + ) + except (OSError, ValueError, subprocess.SubprocessError) as exc: + detail = getattr(exc, "strerror", None) or str(exc) + return _emit_final( + audit, + "launch_error", + EXIT_LAUNCH_ERROR, + "command launch failed", + snapshot, + snapshot.used_bytes, + error=detail, + ) + + previous_handlers = _set_parent_signal_handlers( + _raise_parent_signal + ) + if lease_manager is not None: + lease_manager.start(child, audit) + audit.lease_manager = lease_manager + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + mask_restored = True + audit.emit( + "child_started", + **_state_fields( + state.snapshot, + state.peak_used_bytes, + child, + None, + "active", + "none", + ), + command=list(config.command), + ) + return _monitor_child( + config, + reader, + audit, + child, + state, + signal_group, + group_alive, + child.pulse if isinstance(child, GuardianProcess) else lambda: None, + monotonic, + sleeper, + ) + except ArtifactError as exc: + _set_parent_signal_handlers(signal.SIG_IGN) + audit.disable_component(exc.component) + if child is None: + return _emit_final( + audit, + "lease_error", + EXIT_LEASE_ERROR, + "watchdog artifact initialization failed", + state.snapshot, + state.peak_used_bytes, + error=f"{exc.component}: {exc}", + ) + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "lease_error", + EXIT_LEASE_ERROR, + "watchdog artifact update failed", + signal.SIGTERM, + config.grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + error=f"{exc.component}: {exc}", + ) + except ParentSignal as exc: + if audit.finalized: + return audit.final_exit_code + _set_parent_signal_handlers(signal.SIG_IGN) + assert child is not None + signal_name = signal.Signals(exc.signal_number).name + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "parent_signal", + 128 + exc.signal_number, + f"wrapper received {signal_name}", + exc.signal_number, + config.grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + ) + except Exception as exc: + if audit.finalized: + return audit.final_exit_code + _set_parent_signal_handlers(signal.SIG_IGN) + if child is None: + return _emit_final( + audit, + "internal_error", + EXIT_INTERNAL_ERROR, + "unexpected pre-launch exception", + state.snapshot, + state.peak_used_bytes, + error=f"{type(exc).__name__}: {exc}", + ) + return _graceful_cleanup( + audit, + child, + state.snapshot, + state.peak_used_bytes, + "internal_error", + EXIT_INTERNAL_ERROR, + "unexpected post-launch exception", + signal.SIGTERM, + config.grace_seconds, + signal_group, + group_alive, + monotonic, + sleeper, + error=f"{type(exc).__name__}: {exc}", + ) + finally: + if not mask_restored: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + if previous_handlers: + _restore_parent_signal_handlers(previous_handlers) + if isinstance(child, GuardianProcess): + child.close() + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be greater than zero") + return parsed + + +def _positive_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError("value must be greater than zero") + return parsed + + +def parse_args(argv: Sequence[str]) -> WatchdogConfig: + parser = argparse.ArgumentParser( + description=( + "Launch a command in a new process group and stop it before " + "host-wide memory use reaches the 120 GiB Strix validation ceiling." + ) + ) + parser.add_argument( + "--procfs-root", + type=Path, + default=Path("/proc"), + help="procfs root containing meminfo and swaps (default: /proc)", + ) + parser.add_argument( + "--soft-gib", + type=_positive_int, + default=116, + help="send SIGTERM at this many GiB used (default: 116)", + ) + parser.add_argument( + "--emergency-gib", + type=_positive_int, + default=118, + help=( + "send SIGKILL at this many GiB used (default: 118, leaving " + "a 2 GiB sampling margin below 120 GiB)" + ), + ) + parser.add_argument( + "--grace-seconds", + type=_positive_float, + default=DEFAULT_GRACE_SECONDS, + help="maximum time after SIGTERM before SIGKILL (default: 30)", + ) + parser.add_argument( + "--sample-interval-seconds", + type=_positive_float, + default=DEFAULT_SAMPLE_INTERVAL_SECONDS, + help="procfs sampling interval (default: 1)", + ) + parser.add_argument( + "--lease-path", + type=Path, + help=( + "atomically publish the watchdog-owned lease JSON; requires " + "--heartbeat-path and --audit-path" + ), + ) + parser.add_argument( + "--heartbeat-path", + type=Path, + help=( + "atomically update watchdog heartbeat JSON on every sample; " + "requires --lease-path and --audit-path" + ), + ) + parser.add_argument( + "--audit-path", + type=Path, + help=( + "create a persistent JSONL audit in addition to standard error; " + "requires --lease-path and --heartbeat-path" + ), + ) + parser.add_argument( + "--heartbeat-max-age-seconds", + type=_positive_float, + default=DEFAULT_HEARTBEAT_MAX_AGE_SECONDS, + help=( + "maximum heartbeat age accepted by a matching harness " + "(default: 5)" + ), + ) + parser.add_argument( + "command", + nargs=argparse.REMAINDER, + help="command and arguments, preceded by --", + ) + args = parser.parse_args(argv) + command = tuple(args.command) + if command and command[0] == "--": + command = command[1:] + return WatchdogConfig( + command=command, + procfs_root=args.procfs_root, + soft_bytes=args.soft_gib * GIB, + emergency_bytes=args.emergency_gib * GIB, + grace_seconds=args.grace_seconds, + sample_interval_seconds=args.sample_interval_seconds, + lease_path=args.lease_path, + heartbeat_path=args.heartbeat_path, + audit_path=args.audit_path, + heartbeat_max_age_seconds=args.heartbeat_max_age_seconds, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = tuple(argv if argv is not None else sys.argv[1:]) + if arguments and arguments[0] == "--internal-guardian": + if len(arguments) < 7 or arguments[5] != "--": + return EXIT_LAUNCH_ERROR + try: + return _guardian_main( + int(arguments[1]), + int(arguments[2]), + _positive_float(arguments[3]), + _positive_float(arguments[4]), + tuple(arguments[6:]), + ) + except (OSError, ValueError): + return EXIT_LAUNCH_ERROR + config = parse_args(arguments) + audit = AuditLogger(sys.stderr) + try: + return run_watchdog(config, audit=audit) + except ValueError as exc: + return _emit_final( + audit, + "configuration_error", + EXIT_PROCFS_ERROR, + "invalid configuration", + None, + None, + error=str(exc), + ) + except Exception as exc: + return _emit_final( + audit, + "internal_error", + EXIT_INTERNAL_ERROR, + "unexpected watchdog error", + None, + None, + error=f"{type(exc).__name__}: {exc}", + ) + finally: + audit.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 04d6d3b2afc8..340374ab2cd7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -211,6 +211,26 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) ARGS -a deepseek41 -s 1 ) + find_package(Python3 COMPONENTS Interpreter) + if (Python3_Interpreter_FOUND) + llama_test_cmd( + ${Python3_EXECUTABLE} + NAME test-deepseek41-trace + LABEL main + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test-deepseek41-trace.py + ) + if(BUILD_SHARED_LIBS AND LLAMA_BUILD_TOOLS AND NOT EMSCRIPTEN) + set_property( + TEST test-deepseek41-trace + APPEND PROPERTY ENVIRONMENT + "DSV41_NATIVE_TRACE_BINARY=$" + "DSV41_NATIVE_CONTAINMENT_HELPER=$" + "DSV41_NATIVE_MANIFEST_BINARY=$" + "DSV41_NATIVE_INJECT_LIBRARY=$") + endif() + endif() + set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") file(MAKE_DIRECTORY "${MODEL_DIR}") @@ -280,6 +300,17 @@ llama_build_and_test(test-chat-template.cpp) # debug tool for chat template differential analysis (not registered as a test, run it manually) llama_build(test-chat-analysis.cpp) llama_build_and_test(test-log.cpp) + +find_package(Python3 3.10 COMPONENTS Interpreter QUIET) +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND Python3_Interpreter_FOUND) + llama_test_cmd( + ${Python3_EXECUTABLE} + NAME test-strix-memory-watchdog + LABEL python + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test_strix_memory_watchdog.py + ) +endif() + llama_build_and_test( test-peg-parser.cpp peg-parser/simple-tokenize.cpp diff --git a/tests/test-deepseek41-runtime.cpp b/tests/test-deepseek41-runtime.cpp index 696abea90100..2b2579a3eaaa 100644 --- a/tests/test-deepseek41-runtime.cpp +++ b/tests/test-deepseek41-runtime.cpp @@ -1,5 +1,6 @@ #include "../src/llama-dsv41.h" #include "../src/llama-arch.h" +#include "../tools/deepseek-v41-trace/trace-components.h" #include "ggml-backend.h" #include "ggml-cpu.h" @@ -377,14 +378,27 @@ static void test_graph_contract() { llama_dsv41_graph_trace_name("expert.ids", il) == "dsv41.trace.expert.ids.l" + std::to_string(il), "expert ID trace name is unstable"); + const auto expert_ids = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("expert.ids", il)); + check(expert_ids && expert_ids->component == "expert.ids" && + expert_ids->layer == (int) il && + std::string(expert_ids->semantic_id_space) == "original", + "exporter does not recognize original expert ID trace"); check( llama_dsv41_graph_trace_name("expert.weights", il) == "dsv41.trace.expert.weights.l" + std::to_string(il), "expert weight trace name is unstable"); + const auto expert_weights = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("expert.weights", il)); + check(expert_weights && expert_weights->component == "expert.weights" && + expert_weights->layer == (int) il, + "exporter does not recognize expert weight trace"); check( llama_dsv41_graph_trace_name("attn.source", il) == "dsv41.trace.attn.source.l" + std::to_string(il), "attention source trace name is unstable"); + const auto attention_source = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("attn.source", il)); + check(attention_source && attention_source->component == "attn.source" && + attention_source->layer == (int) il, + "exporter does not recognize attention source trace"); if (il > LLAMA_DSV41_CANDIDATE_SOURCE_LAYER && llama_dsv41_index_source_layer(il) == (int32_t) il) { candidate_trace_layers.push_back(il); @@ -392,6 +406,10 @@ static void test_graph_contract() { llama_dsv41_graph_trace_name("attn.candidates", il) == "dsv41.trace.attn.candidates.l" + std::to_string(il), "attention candidate trace name is unstable"); + const auto candidates = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("attn.candidates", il)); + check(candidates && candidates->component == "attn.candidates" && + candidates->layer == (int) il, + "exporter does not recognize propagated candidate trace"); } } check(ratio_count[0] == 2 && ratio_count[1] == 20 && ratio_count[2] == 18, @@ -405,12 +423,40 @@ static void test_graph_contract() { llama_dsv41_graph_trace_name("attn.candidate_blocks", 20) == "dsv41.trace.attn.candidate_blocks.l20", "candidate block trace name is unstable"); + const auto candidate_blocks = + dsv41_trace_parse_name(llama_dsv41_graph_trace_name("attn.candidate_blocks", 20)); + check(candidate_blocks && candidate_blocks->component == "attn.candidate_blocks" && + candidate_blocks->layer == 20, + "exporter does not recognize candidate block trace"); check( llama_dsv41_graph_trace_name("engram.row_ids", 1) == "dsv41.trace.engram.row_ids.l1" && llama_dsv41_graph_trace_name("engram.row_ids", 14) == "dsv41.trace.engram.row_ids.l14", "Engram row trace names are unstable"); + for (uint32_t layer : { 1u, 14u }) { + const auto engram = dsv41_trace_parse_name(llama_dsv41_graph_trace_name("engram.row_ids", layer)); + check(engram && engram->component == "engram.row_ids" && + engram->layer == (int) layer, + "exporter does not recognize Engram row trace"); + } + expect_throw( + [] { + dsv41_trace_select_name("dsv41.trace.attn.candidates.l20"); + }, + "exporter accepted an unexpected candidate trace layer"); + expect_throw( + [] { + dsv41_trace_select_name("dsv41.trace.attn.candidates.layer24"); + }, + "exporter accepted a malformed trace layer suffix"); + expect_throw( + [] { + dsv41_trace_select_name("dsv41.trace.unknown.l24"); + }, + "exporter accepted an unknown reserved trace tensor name"); + check(!dsv41_trace_select_name("dsv41_attn_candidates_l24"), + "exporter treated an ordinary graph tensor as reserved"); check(llama_dsv41_build_layer_plan(39, { 39 }, 1024).collapses_output, "final layer must preserve streams for carried-pre output collapse"); } diff --git a/tests/test-deepseek41-trace.py b/tests/test-deepseek41-trace.py new file mode 100644 index 000000000000..375a140e22c6 --- /dev/null +++ b/tests/test-deepseek41-trace.py @@ -0,0 +1,8424 @@ +#!/usr/bin/env python3 + +import array +import contextlib +import copy +import hashlib +import importlib.util +import inspect +import io +import json +import os +import shutil +import subprocess +import struct +import sys +import tempfile +import time +import unittest +from argparse import Namespace +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from unittest import mock + +TRACE_DIR = Path(__file__).parents[1] / "tools" / "deepseek-v41-trace" +sys.path.insert(0, str(TRACE_DIR)) +MODULE_PATH = TRACE_DIR / "trace_format.py" +SPEC = importlib.util.spec_from_file_location("dsv41_trace_format", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +trace = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(trace) +import run_llama +import run_ds4 +import run_matrix +import preflight +import verify_ds4_anchors + +trace.APPROVED_WATCHDOGS[trace.WATCHDOG_SCRIPT_SHA256] = trace.WATCHDOG_REVISION +FIXTURE_DS4_EXPORTER_SHA256 = "3" * 64 +TEST_AUTH_ISSUED = int(time.time()) - 60 +TEST_AUTH_EXPIRES = TEST_AUTH_ISSUED + 3600 +TEST_CHALLENGE = "d" * 64 +TEST_RUN_IDS = { + "llama.cpp": "strix-llama-test-run", + "ds4": "apple-ds4-test-run", +} +TEST_CANDIDATE_EXPORTER_POLICY_ID = "test-candidate-exporter" +TEST_DS4_EXPORTER_POLICY_ID = "test-ds4-exporter" +TEST_PROMPT_BUILDER_POLICY_ID = "test-prompt-builder" +NATIVE_TEST_STATE_PACKET_MAX_BYTES = 4096 +NATIVE_TEST_STATE_KEYS = frozenset({ + "caps", + "gids", + "groups", + "no_new_privs", + "pgrp", + "pid", + "seccomp", + "securebits", + "sid", + "uids", +}) + + +def receive_native_test_pidfd_packet(connection, max_payload_bytes): + import array + import socket + + item_size = array.array("i").itemsize + requested_flags = getattr(socket, "MSG_CMSG_CLOEXEC", 0) + data, ancillary, flags, _address = connection.recvmsg( + max_payload_bytes + 1, + socket.CMSG_SPACE(item_size), + requested_flags, + ) + descriptors = [] + try: + for level, kind, content in ancillary: + if level != socket.SOL_SOCKET or kind != socket.SCM_RIGHTS: + raise AssertionError("native test packet contained unexpected ancillary data") + descriptor_bytes = array.array("i") + descriptor_bytes.frombytes(content[:len(content) - len(content) % item_size]) + descriptors.extend(descriptor_bytes) + truncation_flags = ( + getattr(socket, "MSG_TRUNC", 0) | + getattr(socket, "MSG_CTRUNC", 0) + ) + if flags & truncation_flags: + raise AssertionError("native test packet was truncated") + if flags not in {0, requested_flags}: + raise AssertionError(f"native test packet returned unexpected flags {flags}") + if len(data) > max_payload_bytes: + raise AssertionError("native test packet exceeded its bounded payload") + if len(descriptors) != 1: + raise AssertionError("native test packet did not provide exactly one pidfd") + return data, descriptors.pop() + finally: + for descriptor in descriptors: + os.close(descriptor) + + +def decode_native_test_report(data): + if len(data) > NATIVE_TEST_STATE_PACKET_MAX_BYTES: + raise AssertionError("native test report exceeded its bounded payload") + try: + decoded = data.decode("ascii") + except UnicodeDecodeError as error: + raise AssertionError("native test report was not ASCII") from error + label, separator, payload = decoded.partition(":") + if label == "descendant": + if separator or payload: + raise AssertionError("descendant report contained an unexpected payload") + return label, None + if label != "target" or not separator or not payload: + raise AssertionError("native test report had an invalid label or payload") + try: + state = json.loads(payload) + except json.JSONDecodeError as error: + raise AssertionError("native target state was not valid JSON") from error + if not isinstance(state, dict) or set(state) != NATIVE_TEST_STATE_KEYS: + raise AssertionError("native target state schema did not match") + return label, state + + +def finish_native_test_supervisor(supervisor, *, kill_if_running): + try: + if kill_if_running and supervisor.poll() is None: + supervisor.kill() + _stdout, stderr = supervisor.communicate(timeout=5) + return supervisor.returncode, stderr or "" + except subprocess.TimeoutExpired: + if supervisor.poll() is None: + supervisor.kill() + supervisor.communicate(timeout=5) + raise + finally: + for stream in (supervisor.stdin, supervisor.stdout, supervisor.stderr): + if stream is not None and not stream.closed: + stream.close() + +WATCHDOG_EVENTS = [ + { + "timestamp": "1970-01-01T00:00:01.000Z", + "event": "preflight", + "total_bytes": 128 * 1024 * 1024 * 1024, + "available_bytes": 64 * 1024 * 1024 * 1024, + "used_bytes": 64 * 1024 * 1024 * 1024, + "swap_entries": 0, + "peak_used_bytes": 64 * 1024 * 1024 * 1024, + "child_pid": None, + "child_status": "not_started", + "child_returncode": None, + "process_group_id": None, + "process_group_status": "not_created", + "threshold_reason": "none", + "soft_bytes": trace.SOFT_MEMORY_LIMIT, + "emergency_bytes": trace.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": trace.STRICT_MEMORY_LIMIT, + }, + { + "timestamp": "1970-01-01T00:00:01.000Z", + "event": "child_started", + "total_bytes": 128 * 1024 * 1024 * 1024, + "available_bytes": 64 * 1024 * 1024 * 1024, + "used_bytes": 64 * 1024 * 1024 * 1024, + "swap_entries": 0, + "peak_used_bytes": 64 * 1024 * 1024 * 1024, + "child_pid": 456, + "child_status": "running", + "child_returncode": None, + "process_group_id": 455, + "process_group_status": "active", + "threshold_reason": "none", + "command": ["python3", "run_matrix.py"], + }, +] +WATCHDOG_JSONL = "".join( + json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + for event in WATCHDOG_EVENTS +).encode("ascii") +WATCHDOG_JSONL_SHA256 = trace.sha256_bytes(WATCHDOG_JSONL) + +ACCELERATOR_ATTESTATION = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "backend": "ROCm", + "backend_device": "ROCm0", + "backend_description": "AMD Radeon Graphics", + "pci_device_id": "0000:c1:00.0", + "kfd_node": "1", + "gpu_id": 1234, + "gfx_target_version": 110501, + "architecture": "gfx1151", + "source": "linux-kfd-sysfs", +} + +METAL_ACCELERATOR_ATTESTATION = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "backend": "Metal", + "backend_device": "Metal0", + "backend_description": "Apple M3 Ultra", + "architecture": "Apple M3 Ultra", + "metal_registry_id": 0x12345678, + "recommended_max_working_set_bytes": 256 * 1024 * 1024 * 1024, + "unified_memory": True, + "source": "metal-device-query", +} + +DS4_RUNTIME_PROFILE = { + "name": "sibling-lib", + "components": ["ds4-runtime", "metal-backend"], + "selected_backend_component": "metal-backend", +} +DS4_RUNTIME_RECEIPT = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": trace.DS4_REVISION, + "profile": "sibling-lib", + "components": [ + { + "component": "ds4-runtime", + "filename": "libds4-runtime.dylib", + "sha256": "a" * 64, + "revision": trace.DS4_REVISION, + }, + { + "component": "metal-backend", + "filename": "libds4-metal.dylib", + "sha256": "b" * 64, + "revision": None, + }, + ], +} +DS4_RUNTIME_LIBRARIES = [ + { + "component": component["component"], + "filename": component["filename"], + "path": f"/Users/oracle/ds4-install/lib/{component['filename']}", + "sha256": component["sha256"], + "role": f"runtime:{component['component']}", + "revision": component["revision"], + } + for component in DS4_RUNTIME_RECEIPT["components"] +] +DS4_RUNTIME_LIBRARIES.sort(key=lambda item: item["path"]) +DS4_RUNTIME_BUILD = { + "revision": trace.DS4_REVISION, + "path": "/Users/oracle/ds4-install/bin/ds4-trace", + "sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": DS4_RUNTIME_PROFILE, + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(DS4_RUNTIME_RECEIPT).encode("ascii")), + "runtime_libraries": DS4_RUNTIME_LIBRARIES, + "runtime_libraries_post": copy.deepcopy(DS4_RUNTIME_LIBRARIES), +} +DS4_INSTALL_TRUST = { + "format": "dsv41-install-trust", + "version": 1, + "install_root": "/Users/oracle/ds4-install", + "owner_uid": 0, + "execution_uid": 501, + "directories": [ + { + "path": path, + "device": 1, + "inode": index, + "owner_uid": 0, + "mode": 0o555, + "effective_write_access": False, + "acl_entries": False, + } + for index, path in enumerate( + ( + "/", + "/Users", + "/Users/oracle", + "/Users/oracle/ds4-install", + "/Users/oracle/ds4-install/bin", + "/Users/oracle/ds4-install/lib", + ), + 1, + ) + ], + "files": [ + { + "path": path, + "device": 1, + "inode": index, + "owner_uid": 0, + "mode": 0o555, + "link_count": 1, + "byte_count": index, + "modified_ns": index, + "changed_ns": index, + "sha256": digest, + "effective_write_access": False, + "acl_entries": False, + } + for index, (path, digest) in enumerate( + sorted(( + ("/Users/oracle/ds4-install/bin/ds4-trace", FIXTURE_DS4_EXPORTER_SHA256), + ("/Users/oracle/ds4-install/lib/libds4-runtime.dylib", "a" * 64), + ("/Users/oracle/ds4-install/lib/libds4-metal.dylib", "b" * 64), + )), + 100, + ) + ], +} +DS4_EXPORTER_POLICY = { + "runtime": "ds4", + "repository": trace.DS4_REPOSITORY, + "revision": trace.DS4_REVISION, + "install_root": "/Users/oracle/ds4-install", + "install_owner_uid": 0, + "executable_path": "/Users/oracle/ds4-install/bin/ds4-trace", + "executable_sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": DS4_RUNTIME_PROFILE, + "runtime_receipt": DS4_RUNTIME_RECEIPT, +} +_DS4_POLICY, DS4_EXPORTER_POLICY_SHA256 = trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: DS4_EXPORTER_POLICY}, +) +DS4_INSTALL_TRUST_SHA256 = trace.install_trust_sha256(DS4_INSTALL_TRUST) +DS4_RUNTIME_BUILD_SHA256 = trace.runtime_build_evidence_sha256( + DS4_RUNTIME_BUILD, + DS4_EXPORTER_POLICY, + label="ds4 exporter", +) + + +def storage_record(path: str) -> dict[str, object]: + model_storage = path.startswith("/mnt/models") + mount_point = "/mnt/models" if model_storage else "/home" + source = "/dev/nvme1n1" if model_storage else "/dev/nvme0n1p3[/home]" + device_number = "259:0" if model_storage else "259:3" + nvme_device = "nvme1n1" if model_storage else "nvme0n1" + return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "storage_kind": "linux-nvme", + "resolved_path": path, + "existing_path": path, + "mount_point": mount_point, + "filesystem_type": "xfs" if model_storage else "btrfs", + "mount_source": source, + "device_number": device_number, + "block_device_path": f"/sys/devices/pci/block/{nvme_device}", + "nvme_device": nvme_device, + "rotational": False, + "source": "linux-mountinfo-sysfs", + } + + +STORAGE_ATTESTATION = { + "model": storage_record("/mnt/models/model.gguf"), + "prompt": storage_record("/home/prompt.txt"), + "output": storage_record("/home"), + "repository": storage_record("/home/repo"), + "temporary_directory": storage_record("/home/tmp"), +} + +def metal_storage_record(path: str, mount_point: str = "/Users") -> dict[str, object]: + return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "storage_kind": "darwin-local-solid-state", + "resolved_path": path, + "existing_path": path, + "mount_point": mount_point, + "filesystem_type": "apfs", + "device_identifier": "disk3s1", + "parent_whole_disk": "disk3", + "bus_protocol": "Apple Fabric", + "filesystem_device": 1, + "internal": True, + "solid_state": True, + "source": "diskutil-info-plist", + } + + +DS4_STORAGE_ATTESTATION = { + "model": metal_storage_record("/Users/oracle/model.gguf"), + "prompt": metal_storage_record("/Users/oracle/prompt.txt"), + "output": metal_storage_record("/Users/oracle/output"), + "repository": metal_storage_record("/Users/oracle/repo"), + "runtime_checkout": metal_storage_record("/Users/oracle/ds4"), + "temporary_directory": metal_storage_record("/Users/oracle/tmp"), + "runner_executable": metal_storage_record("/usr/bin/python3", "/"), + "runner_script": metal_storage_record("/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py"), + "exporter": metal_storage_record("/Users/oracle/ds4-install/bin/ds4-trace"), +} + +DS4_HOST_ATTESTATION = { + "format": "dsv41-host-attestation", + "version": 1, + "runtime_kind": "apple-metal", + "platform": "macos", + "machine": "arm64", + "hardware_model": "Mac14,8", + "os_version": "15.6", + "memory_bytes": 256 * 1024 * 1024 * 1024, + "source": "darwin-sysctl", +} + +DS4_RUNNER_ATTESTATION = { + "format": "dsv41-runner-ownership", + "version": 1, + "runtime_kind": "apple-metal", + "source": "python-subprocess", + "runner_pid": 100, + "runner_parent_pid": 99, + "runner_uid": 501, + "runner_executable": "/usr/bin/python3", + "runner_executable_sha256": "1" * 64, + "runner_script": "/Users/oracle/repo/tools/deepseek-v41-trace/run_ds4.py", + "runner_script_sha256": "2" * 64, + "exporter_path": "/Users/oracle/ds4-install/bin/ds4-trace", + "exporter_sha256": FIXTURE_DS4_EXPORTER_SHA256, + "exporter_approval_id": TEST_DS4_EXPORTER_POLICY_ID, + "exporter_approval_sha256": DS4_EXPORTER_POLICY_SHA256, + "exporter_install_trust_sha256": DS4_INSTALL_TRUST_SHA256, + "exporter_runtime_build_sha256": DS4_RUNTIME_BUILD_SHA256, + "exporter_runtime_profile": DS4_RUNTIME_PROFILE, + "exporter_runtime_receipt_sha256": DS4_RUNTIME_BUILD["runtime_receipt_sha256"], + "producer_revision": trace.DS4_REVISION, + "verifier_revision": "a" * 40, + "checkout_path": "/Users/oracle/ds4", + "checkout_revision": trace.DS4_REVISION, + "command_sha256": "4" * 64, +} + +AUDIT_RECORDS = { + "memory": { + "created_unix": 1, + "kind": "memory", + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, + "data": {"mem_total_bytes": 128, "mem_available_bytes": 64, "mem_used_bytes": 64}, + "storage": STORAGE_ATTESTATION, + "storage_policy": json.loads(json.dumps(trace.NO_EXTERNAL_STATE_STORAGE)), + "accelerator": dict(ACCELERATOR_ATTESTATION), + }, + "swap": { + "created_unix": 1, + "kind": "swap", + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, + "data": {"enabled": False, "entries": []}, + }, + "watchdog": { + "created_unix": 1, + "kind": "watchdog", + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, + "data": { + "format": trace.WATCHDOG_LEASE_FORMAT, + "version": trace.WATCHDOG_VERSION, + "lease_id": "1" * 32, + "state": "active", + "file_device": 1, + "file_inode": 2, + "file_uid": 1000, + "file_mode": 0o600, + "lease_path": "/run/user/123/watchdog.lease", + "watchdog_pid": 123, + "watchdog_start_time_utc": "1970-01-01T00:00:01.000Z", + "watchdog_start_time_ticks": 456, + "watchdog_command": "python3 /repo/scripts/strix_memory_watchdog.py", + "watchdog_command_sha256": "7" * 64, + "watchdog_executable_path": "/usr/bin/python3", + "watchdog_script_path": "/repo/scripts/strix_memory_watchdog.py", + "watchdog_script_sha256": trace.WATCHDOG_SCRIPT_SHA256, + "watchdog_revision": trace.WATCHDOG_REVISION, + "soft_bytes": trace.SOFT_MEMORY_LIMIT, + "emergency_bytes": trace.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": trace.STRICT_MEMORY_LIMIT, + "grace_seconds": 30.0, + "sample_interval_seconds": 1.0, + "procfs_root": "/proc", + "guardian_pid": 455, + "child_pid": 456, + "child_process_group_id": 455, + "command": ["python3", "run_matrix.py"], + "child_command_sha256": trace.sha256_bytes(b'["python3","run_matrix.py"]'), + "heartbeat_path": "/run/user/123/watchdog.heartbeat", + "heartbeat_unix": 1, + "max_heartbeat_age_seconds": 5.0, + "audit_live_path": "/run/user/123/watchdog.jsonl", + "audit_device": 1, + "audit_inode": 2, + "audit_uid": 1000, + "audit_mode": 0o600, + "audit_fd": 3, + "audit_sha256": WATCHDOG_JSONL_SHA256, + "namespace_authority": { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": 9, + "host_procfs_root": "/proc", + "watchdog_pid": 123, + "watchdog_process_group_id": 122, + "watchdog_start_time_ticks": 456, + "watchdog_executable_path": "/usr/bin/python3", + "watchdog_command_sha256": "7" * 64, + "guardian_pid": 455, + "child_pid": 456, + "child_process_group_id": 455, + }, + "audit": { + "path": "", + "sha256": WATCHDOG_JSONL_SHA256, + "event_count": len(WATCHDOG_EVENTS), + }, + }, + }, +} + +DS4_AUDIT_RECORDS = { + "memory": { + "created_unix": 1, + "kind": "memory", + "environment": {}, + "data": { + "mem_total_bytes": DS4_HOST_ATTESTATION["memory_bytes"], + "mem_available_bytes": 128 * 1024 * 1024 * 1024, + "mem_used_bytes": 128 * 1024 * 1024 * 1024, + }, + "storage": DS4_STORAGE_ATTESTATION, + "storage_policy": json.loads(json.dumps(trace.NO_EXTERNAL_STATE_STORAGE)), + "accelerator": dict(METAL_ACCELERATOR_ATTESTATION), + "host": DS4_HOST_ATTESTATION, + }, + "swap": { + "created_unix": 1, + "kind": "swap", + "environment": {}, + "data": { + "source": "darwin-sysctl-vm.swapusage", + "total_bytes": 0, + "used_bytes": 0, + "free_bytes": 0, + }, + }, + "runner": { + "created_unix": 1, + "kind": "runner", + "environment": {}, + "data": DS4_RUNNER_ATTESTATION, + }, +} + + +def audit_bytes(kind: str, phase: str, runtime: str = "llama.cpp") -> bytes: + records = DS4_AUDIT_RECORDS if runtime == "ds4" else AUDIT_RECORDS + record = json.loads(json.dumps(records[kind])) + if kind == "watchdog": + record["data"]["audit"]["path"] = f"audits/{phase}/{WATCHDOG_JSONL_SHA256}.jsonl" + return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + +def replace_audit_record(root: Path, phase: str, kind: str, record: dict[str, object]) -> None: + data = (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + digest = trace.sha256_bytes(data) + path = root / "audits" / phase / f"{digest}.json" + path.write_bytes(data) + manifest_path = root / trace.MANIFEST_NAME + manifest_record = json.loads(manifest_path.read_text(encoding="ascii")) + manifest_record["audits"][phase][kind] = { + "path": f"audits/{phase}/{digest}.json", + "sha256": digest, + "created_unix": record["created_unix"], + } + manifest_path.write_text( + json.dumps(manifest_record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + + +def replace_watchdog_events(root: Path, phase: str, events: list[dict[str, object]]) -> None: + records = [] + for event in events: + record = json.loads(json.dumps(event)) + records.append(record) + data = b"".join( + (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + for record in records + ) + digest = trace.sha256_bytes(data) + path = root / "audits" / phase / f"{digest}.jsonl" + path.write_bytes(data) + record = json.loads(json.dumps(AUDIT_RECORDS["watchdog"])) + record["data"]["audit_sha256"] = digest + record["data"]["audit"]["path"] = f"audits/{phase}/{digest}.jsonl" + record["data"]["audit"]["sha256"] = digest + record["data"]["audit"]["event_count"] = len(records) + replace_audit_record(root, phase, "watchdog", record) + + +def fixture_containment_helper(revision: str, digest: str = "d" * 64) -> dict[str, object]: + return { + "format": "dsv41-containment-helper", + "version": 2, + "revision": revision, + "filename": "llama-deepseek-v41-containment-helper", + "sha256": digest, + "launcher_policy": "zero-supplementary-groups-v1", + "supplementary_groups": [], + } + + +def fixture_prompt_builder_policy( + prompt: bytes, + *, + context: int = 3, + decode_steps: int = 1, + builder_path: str = "/home/repo/build/bin/llama-deepseek-v41-prompt-builder", + builder_sha256: str = "8" * 64, + source_root: str = "/home/repo") -> dict[str, object]: + runtime_components = [ + ("ggml", "libggml.so", "4", None), + ("ggml-base", "libggml-base.so", "5", "a" * 40), + ("ggml-hip", "libggml-hip.so", "6", None), + ("llama", "libllama.so", "7", None), + ("llama-common", "libllama-common.so", "9", "a" * 40), + ] + runtime_profile = { + "name": "sibling-lib", + "components": sorted(component for component, *_rest in runtime_components), + "selected_backend_component": "ggml-hip", + } + return { + "runtime": "llama.cpp", + "runtime_profile": runtime_profile, + "repository": trace.REPOSITORY, + "revision": "a" * 40, + "install_root": str(Path(builder_path).parent.parent), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "executable_path": builder_path, + "executable_sha256": builder_sha256, + "containment_helper": fixture_containment_helper("a" * 40), + "source_root": source_root, + "runtime_receipt": { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "a" * 40, + "profile": "sibling-lib", + "components": [ + { + "component": component, + "filename": filename, + "sha256": digest * 64, + "revision": revision, + } + for component, filename, digest, revision in runtime_components + ], + }, + "model_sha256": trace.MODEL_SHA256, + "corpora": dict(trace.CORPUS_SHA256), + "tokenizer": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": True, + }, + "prompts": [{ + "corpus_name": "correctness-prose.txt", + "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "context": context, + "decode_steps": decode_steps, + "target_tokens": context - decode_steps, + "prompt_sha256": trace.sha256_bytes(prompt), + "prompt_byte_count": len(prompt), + }], + } + + +def materialize_policy_runtime(policy: dict[str, object]) -> None: + library_root = Path(policy["install_root"]) / "lib" + library_root.mkdir(parents=True, exist_ok=True) + for component in policy["runtime_receipt"]["components"]: + path = library_root / component["filename"] + path.write_bytes(component["component"].encode("ascii")) + component["sha256"] = trace.sha256_file(path) + path.chmod(0o555) + executable = Path(policy["executable_path"]) + containment_helper = policy.get("containment_helper") + if containment_helper is not None: + helper = Path(policy["install_root"]) / "bin" / containment_helper["filename"] + helper.write_bytes(b"containment-helper") + helper.chmod(0o555) + containment_helper["sha256"] = trace.sha256_file(helper) + if executable.exists(): + executable.chmod(0o555) + library_root.chmod(0o555) + executable.parent.chmod(0o555) + Path(policy["install_root"]).chmod(0o555) + + +def materialize_ds4_exporter_policy(root: Path) -> tuple[dict[str, object], Path]: + install = root / "ds4-install" + exporter = install / "bin" / "ds4-trace" + exporter.parent.mkdir(parents=True) + exporter.write_text("#!/bin/sh\nprintf 'approved\\n'\n", encoding="ascii") + exporter.chmod(0o555) + policy = copy.deepcopy(DS4_EXPORTER_POLICY) + policy["install_root"] = str(install) + policy["install_owner_uid"] = os.geteuid() if hasattr(os, "geteuid") else 0 + policy["executable_path"] = str(exporter) + policy["executable_sha256"] = trace.sha256_file(exporter) + materialize_policy_runtime(policy) + return policy, exporter + + +def fixture_install_trust(policy: dict[str, object]) -> dict[str, object]: + install_root = Path(policy["install_root"]) + paths = { + policy["executable_path"]: policy["executable_sha256"], + **{ + str(install_root / "lib" / component["filename"]): component["sha256"] + for component in policy["runtime_receipt"]["components"] + }, + } + containment_helper = policy.get("containment_helper") + if containment_helper is not None: + paths[str(install_root / "bin" / containment_helper["filename"])] = ( + containment_helper["sha256"]) + directory_paths = set() + for path in paths: + current = Path(path).parent + while True: + directory_paths.add(str(current)) + if current == Path(current.anchor): + break + current = current.parent + owner_uid = policy["install_owner_uid"] + directories = [] + for index, path in enumerate(sorted(directory_paths), 1): + in_install = path == str(install_root) or install_root in Path(path).parents + directories.append({ + "path": path, + "device": 1, + "inode": index, + "owner_uid": owner_uid if in_install else 0, + "mode": 0o555, + "effective_write_access": False, + "acl_entries": False, + }) + files = [] + for index, (path, digest) in enumerate(sorted(paths.items()), 100): + files.append({ + "path": path, + "device": 1, + "inode": index, + "owner_uid": owner_uid, + "mode": 0o555, + "link_count": 1, + "byte_count": index, + "modified_ns": index, + "changed_ns": index, + "sha256": digest, + "effective_write_access": False, + "acl_entries": False, + }) + return { + "format": "dsv41-install-trust", + "version": 1, + "install_root": str(install_root), + "owner_uid": owner_uid, + "execution_uid": owner_uid + 1 if owner_uid != 0 else 1, + "directories": directories, + "files": files, + } + + +def fixture_runtime_build(policy: dict[str, object]) -> dict[str, object]: + libraries = [] + for component in policy["runtime_receipt"]["components"]: + name = component["component"] + libraries.append({ + "component": name, + "filename": component["filename"], + "path": f"{policy['install_root']}/lib/{component['filename']}", + "sha256": component["sha256"], + "role": { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + }.get(name, f"runtime:{name}"), + "revision": component["revision"], + }) + libraries.sort(key=lambda item: item["path"]) + return { + "revision": policy["revision"], + "path": policy["executable_path"], + "sha256": policy["executable_sha256"], + "runtime_profile": policy["runtime_profile"], + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(policy["runtime_receipt"]).encode("ascii")), + "runtime_libraries": libraries, + "runtime_libraries_post": copy.deepcopy(libraries), + } + + +@contextlib.contextmanager +def isolated_test_install_trust(*, process_containment: bool = True): + modules = (trace, sys.modules["trace_format"]) + with contextlib.ExitStack() as stack: + for module in modules: + stack.enter_context(mock.patch.object( + module, + "_execution_uid", + return_value=(os.geteuid() if hasattr(os, "geteuid") else 0) + 1, + )) + stack.enter_context(mock.patch.object( + module, "_path_is_writable_by_execution_identity", return_value=False)) + stack.enter_context(mock.patch.object( + module, "_has_access_control_entries", return_value=False)) + if process_containment and sys.platform == "darwin": + stack.enter_context(module._test_only_process_group_containment()) + yield + + +def provenance_bytes( + prompt: bytes = b"abc", + *, + context: int = 3, + decode_steps: int = 1) -> bytes: + policy = fixture_prompt_builder_policy(prompt, context=context, decode_steps=decode_steps) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + trust = fixture_install_trust(policy) + runtime_build = fixture_runtime_build(policy) + source_root_lexical = Path(policy["source_root"]) + source_root_resolved = source_root_lexical.resolve(strict=False) + corpus_lexical = source_root_lexical / "tests" / "corpus" / "correctness-prose.txt" + corpus_resolved = source_root_resolved / "tests" / "corpus" / "correctness-prose.txt" + record = { + "format": "dsv41-prompt-provenance", + "version": 2, + "corpus_name": "correctness-prose.txt", + "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "corpus_path": str(corpus_resolved), + "corpus_lexical_path": str(corpus_lexical), + "corpus_resolved_path": str(corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), + "model_sha256": trace.MODEL_SHA256, + "prompt_sha256": trace.sha256_bytes(prompt), + "prompt_byte_count": len(prompt), + "context": context, + "decode_steps": decode_steps, + "target_tokens": context - decode_steps, + "actual_tokens": context - decode_steps, + "builder_approval_id": TEST_PROMPT_BUILDER_POLICY_ID, + "builder_approval_sha256": policy_sha256, + "builder_path": policy["executable_path"], + "builder_sha256": "8" * 64, + "builder_revision": "a" * 40, + "builder_runtime_profile": policy["runtime_profile"], + "tokenizer": policy["tokenizer"], + "builder_runtime_build": runtime_build, + "builder_runtime_build_sha256": trace.runtime_build_evidence_sha256( + runtime_build, policy, label="prompt builder"), + "builder_install_trust": trust, + "builder_install_trust_sha256": trace.install_trust_sha256(trust), + } + return (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + +def manifest( + runtime: str = "llama.cpp", + prompt: bytes = b"abc", + *, + context: int = 3, + decode_steps: int = 1) -> dict: + provenance_sha256 = trace.sha256_bytes( + provenance_bytes(prompt, context=context, decode_steps=decode_steps)) + is_ds4 = runtime == "ds4" + storage = DS4_STORAGE_ATTESTATION if is_ds4 else STORAGE_ATTESTATION + audit_kinds = ("memory", "swap", "runner") if is_ds4 else ("memory", "swap", "watchdog") + runtime_components = [ + ("ggml", "libggml.so", "4", "runtime:ggml", None), + ("ggml-base", "libggml-base.so", "5", "ggml", "a" * 40), + ("ggml-hip", "libggml-hip.so", "6", "selected-backend", None), + ("llama", "libllama.so", "7", "llama", None), + ("llama-common", "libllama-common.so", "9", "build-info", "a" * 40), + ] + runtime_libraries = [ + { + "component": component, + "filename": filename, + "path": f"/home/repo/build/lib/{filename}", + "sha256": digest * 64, + "role": role, + "revision": revision, + } + for component, filename, digest, role, revision in runtime_components + ] + runtime_receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "a" * 40, + "profile": "sibling-lib", + "components": [ + { + "component": component, + "filename": filename, + "sha256": digest * 64, + "revision": revision, + } + for component, filename, digest, _role, revision in runtime_components + ], + } + result = { + "runtime": runtime, + "revision": trace.DS4_REVISION if is_ds4 else "a" * 40, + "build": ( + { + "compiler": "clang", + "target": "arm64-apple-darwin", + "path": DS4_RUNTIME_BUILD["path"], + "sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": copy.deepcopy(DS4_RUNTIME_PROFILE), + "runtime_receipt_sha256": DS4_RUNTIME_BUILD["runtime_receipt_sha256"], + "runtime_libraries": copy.deepcopy(DS4_RUNTIME_LIBRARIES), + "runtime_libraries_post": copy.deepcopy(DS4_RUNTIME_LIBRARIES), + "runtime_module_monitor": { + "mechanism": "dyld-add-image", + "checked_after_trace": True, + "project_additions": [], + }, + } + if is_ds4 + else { + "number": 1, + "info": "test", + "compiler": "clang", + "target": "arm64-apple-darwin", + "path": "/home/repo/build/bin/llama-deepseek-v41-trace", + "sha256": "3" * 64, + "runtime_profile": { + "name": "sibling-lib", + "components": [component for component, *_rest in runtime_components], + "selected_backend_component": "ggml-hip", + }, + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(runtime_receipt).encode("ascii")), + "runtime_libraries": sorted(runtime_libraries, key=lambda library: library["path"]), + "runtime_libraries_post": sorted( + copy.deepcopy(runtime_libraries), key=lambda library: library["path"]), + "runtime_module_monitor": { + "mechanism": "pre-post-snapshot", + "checked_after_trace": True, + "project_additions": [], + }, + } + ), + "model": { + "path": storage["model"]["resolved_path"], + "sha256": trace.MODEL_SHA256, + "byte_count": 123, + "architecture": "deepseek41", + }, + "accelerator": dict(METAL_ACCELERATOR_ATTESTATION if is_ds4 else ACCELERATOR_ATTESTATION), + "prompt": { + "path": storage["prompt"]["resolved_path"], + "sha256": trace.sha256_bytes(prompt), + "byte_count": len(prompt), + "corpus_name": "correctness-prose.txt", + "corpus_sha256": trace.CORPUS_SHA256["correctness-prose.txt"], + "target_tokens": context - decode_steps, + "provenance": { + "path": f"provenance/{provenance_sha256}.json", + "sha256": provenance_sha256, + }, + }, + "config": { + "context": context, + "decode_steps": decode_steps, + "deepseek41": { + "layer_count": 40, + "vocab_size": 129280, + "engram_layers": [1, 14], + "engram_rows_per_token": 24, + "expert_count": 384, + "experts_used": 6, + "candidate_source_layer": 20, + "candidate_topk_blocks": 2048, + "candidate_block_size": 8, + "index_top_k": 512, + "raw_attention_layers": list(trace.RAW_ATTENTION_LAYERS), + "raw_attention_width": trace.RAW_ATTENTION_WIDTH, + "candidate_propagation_layers": [24, 28, 32, 36], + }, + }, + "paths": { + label: record["resolved_path"] + for label, record in storage.items() + }, + "storage_policy": json.loads(json.dumps(trace.NO_EXTERNAL_STATE_STORAGE)), + "comparison": { + "tokens": "exact", + "engram_rows": "exact", + "expert_ids": "exact-original-id-space", + "expert_weights": "byte-identical-f32", + "attention_candidates": "exact", + "logits": "byte-identical-f32", + }, + "expected": { + "prompt_tokens": context - decode_steps, + "decode_steps": decode_steps, + "components": { + "prompt.bytes": {"layers": None, "input": "tokens"}, + "prompt.tokens": {"layers": None, "input": "tokens"}, + "engram.row_ids": {"layers": [1, 14], "prefill": "tokens", "decode": "steps"}, + "expert.ids": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "expert.weights": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.source": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.candidate_blocks": {"layers": [20], "prefill": "tokens", "decode": "steps"}, + "attn.candidates": {"layers": [24, 28, 32, 36], "prefill": "tokens", "decode": "steps"}, + "logits.prefill": {"layers": None, "prefill": "final"}, + "logits.decode": {"layers": None, "decode": "steps"}, + "decode.greedy_token": {"layers": None, "decode": "steps"}, + }, + }, + "environment": { + "system_info": "Linux test system" if runtime == "llama.cpp" else "macOS test system", + "command": "test command", + }, + "audits": { + phase: { + kind: { + "path": f"audits/{phase}/{trace.sha256_bytes(audit_bytes(kind, phase, runtime))}.json", + "sha256": trace.sha256_bytes(audit_bytes(kind, phase, runtime)), + "created_unix": 1, + } + for kind in audit_kinds + } + for phase in ("pre", "post") + }, + } + if not is_ds4: + result["config"].update({ + "batch": trace.ADMITTED_BATCH, + "ubatch": trace.ADMITTED_UBATCH, + "kv_type_k": "f16", + "kv_type_v": "f16", + "flash_attention": True, + "expert_cache_slots": trace.REQUIRED_EXPERT_SLOTS, + "expert_cache_bytes": trace.REQUIRED_EXPERT_CACHE_BYTES, + "device": "ROCm0", + "device_architecture": "gfx1151", + "device_pci_id": "0000:c1:00.0", + "gpu_layers": 99, + "load_mode": 0, + "tokenizer": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": True, + }, + "model_file_identity": { + "format": "dsv41-model-file-identity", + "version": 1, + "path": storage["model"]["resolved_path"], + "device": 1, + "inode": 2, + "owner_uid": 1000, + "owner_gid": 1000, + "mode": 0o444, + "link_count": 1, + "byte_count": 123, + "modified_ns": 1, + "changed_ns": 1, + "status_flags": 0, + "source_descriptor_flags": 1, + "target_descriptor_flags": 0, + "sha256": trace.MODEL_SHA256, + }, + "watchdog_namespace": { + "format": "dsv41-watchdog-namespace-binding", + "version": 1, + "authority": "inherited-pidfd", + "host_watchdog_pid": 123, + "host_watchdog_process_group_id": 122, + "host_watchdog_start_time_ticks": 456, + "local_pid": 2, + "local_parent_pid": 1, + "local_process_group_id": 2, + "local_session_id": 2, + "namespace_pids": [1234, 2], + "private_procfs": True, + }, + }) + result["candidate"] = { + "repository": trace.REPOSITORY, + "revision": "a" * 40, + "base_revision": "b" * 40, + "diff_sha256": "c" * 64, + "executable_path": result["build"]["path"], + "executable_sha256": "3" * 64, + "runtime_libraries_sha256": trace.sha256_bytes( + trace.canonical_json({ + "pre": result["build"]["runtime_libraries"], + "post": result["build"]["runtime_libraries_post"], + }).encode("ascii")), + "runtime_receipt_sha256": result["build"]["runtime_receipt_sha256"], + } + else: + result["host"] = dict(DS4_HOST_ATTESTATION) + result["oracle"] = { + "repository": trace.DS4_REPOSITORY, + "revision": trace.DS4_REVISION, + "verifier_revision": "a" * 40, + "executable_path": DS4_RUNTIME_BUILD["path"], + "executable_sha256": FIXTURE_DS4_EXPORTER_SHA256, + "runtime_profile": copy.deepcopy(DS4_RUNTIME_PROFILE), + "runtime_build_sha256": DS4_RUNTIME_BUILD_SHA256, + "runtime_libraries_sha256": trace.sha256_bytes( + trace.canonical_json({ + "pre": DS4_RUNTIME_LIBRARIES, + "post": DS4_RUNTIME_LIBRARIES, + }).encode("ascii")), + "runtime_receipt_sha256": DS4_RUNTIME_BUILD["runtime_receipt_sha256"], + "exporter_approval_id": TEST_DS4_EXPORTER_POLICY_ID, + "exporter_approval_sha256": DS4_EXPORTER_POLICY_SHA256, + "install_trust": copy.deepcopy(DS4_INSTALL_TRUST), + "install_trust_sha256": DS4_INSTALL_TRUST_SHA256, + } + result["config"]["prefill_chunk"] = trace.ADMITTED_UBATCH + result["config"]["device_backend"] = "Metal" + result["config"]["device_registry_id"] = METAL_ACCELERATOR_ATTESTATION["metal_registry_id"] + prompt_policy = fixture_prompt_builder_policy( + prompt, context=context, decode_steps=decode_steps) + _prompt_policy, prompt_policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: prompt_policy}, + ) + prompt_trust = fixture_install_trust(prompt_policy) + approvals = { + "prompt_builder": trace.approval_binding( + "prompt_builder", + TEST_PROMPT_BUILDER_POLICY_ID, + prompt_policy_sha256, + trace.install_trust_sha256(prompt_trust), + ), + } + if is_ds4: + approvals["ds4_exporter"] = trace.approval_binding( + "ds4_exporter", + TEST_DS4_EXPORTER_POLICY_ID, + DS4_EXPORTER_POLICY_SHA256, + DS4_INSTALL_TRUST_SHA256, + ) + else: + candidate_policy = { + "runtime": "llama.cpp", + "repository": trace.REPOSITORY, + "revision": result["candidate"]["revision"], + "base_revision": result["candidate"]["base_revision"], + "diff_sha256": result["candidate"]["diff_sha256"], + "install_root": "/home/repo/build", + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "executable_path": result["candidate"]["executable_path"], + "executable_sha256": result["candidate"]["executable_sha256"], + "containment_helper": fixture_containment_helper(result["candidate"]["revision"]), + "runtime_profile": copy.deepcopy(result["build"]["runtime_profile"]), + "runtime_receipt": runtime_receipt, + } + _candidate_policy, candidate_policy_sha256 = trace.candidate_exporter_approval( + TEST_CANDIDATE_EXPORTER_POLICY_ID, + policies={TEST_CANDIDATE_EXPORTER_POLICY_ID: candidate_policy}, + ) + result["candidate"]["exporter_approval_id"] = TEST_CANDIDATE_EXPORTER_POLICY_ID + result["candidate"]["exporter_approval_sha256"] = candidate_policy_sha256 + candidate_trust = fixture_install_trust(candidate_policy) + result["candidate"]["install_trust"] = candidate_trust + result["candidate"]["install_trust_sha256"] = trace.install_trust_sha256(candidate_trust) + approvals["candidate_exporter"] = trace.approval_binding( + "candidate_exporter", + TEST_CANDIDATE_EXPORTER_POLICY_ID, + candidate_policy_sha256, + trace.install_trust_sha256(candidate_trust), + ) + result["authorization"] = trace.execution_authorization( + lane=trace.ORACLE_LANE if is_ds4 else trace.CANDIDATE_LANE, + challenge=TEST_CHALLENGE, + run_id=TEST_RUN_IDS[runtime], + issued_unix=TEST_AUTH_ISSUED, + expires_unix=TEST_AUTH_EXPIRES, + approval_policy_sha256="e" * 64, + verifier_revision="a" * 40, + tokenizer_policy_sha256_value=trace.tokenizer_policy_sha256(prompt_policy["tokenizer"]), + approvals=approvals, + ) + return result + + +def add_required_events(writer: object, logits: bytes | None = None, prompt: bytes = b"abc") -> None: + runtime = writer.manifest["runtime"] + audit_kinds = ("memory", "swap", "runner") if runtime == "ds4" else ("memory", "swap", "watchdog") + for phase in ("pre", "post"): + audit_root = writer.root / "audits" / phase + audit_root.mkdir(parents=True, exist_ok=True) + for kind in audit_kinds: + data = audit_bytes(kind, phase, runtime) + (audit_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) + if runtime == "llama.cpp": + (audit_root / f"{WATCHDOG_JSONL_SHA256}.jsonl").write_bytes(WATCHDOG_JSONL) + provenance_root = writer.root / "provenance" + provenance_root.mkdir(exist_ok=True) + data = provenance_bytes( + prompt, + context=writer.manifest["config"]["context"], + decode_steps=writer.manifest["config"]["decode_steps"], + ) + (provenance_root / f"{trace.sha256_bytes(data)}.json").write_bytes(data) + writer.add_event( + component="prompt.bytes", + phase="input", + step=0, + token_start=0, + token_count=2, + layer=None, + dtype="bytes", + shape=[len(prompt)], + data=prompt, + ) + writer.add_event( + component="prompt.tokens", + phase="input", + step=0, + token_start=0, + token_count=2, + layer=None, + dtype="i32", + shape=[2], + data=struct.pack(" None: + events_path = root / trace.EVENTS_NAME + events = [json.loads(line) for line in events_path.read_text(encoding="ascii").splitlines()] + event = next( + item for item in events + if item["component"] == component and item["phase"] == phase and item["layer"] == layer + ) + digest = trace.sha256_bytes(data) + blob = root / trace.BLOBS_DIR / f"{digest}.bin" + blob.write_bytes(data) + event.update({ + "shape": shape, + "byte_count": len(data), + "sha256": digest, + "blob": f"{trace.BLOBS_DIR}/{digest}.bin", + }) + events_path.write_text( + "".join(json.dumps(item, sort_keys=True, separators=(",", ":")) + "\n" for item in events), + encoding="ascii", + ) + + +class TraceFormatTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls._signing_directory = tempfile.TemporaryDirectory() + cls.ssh_keygen = trace.trusted_ssh_keygen_path() + cls.signing_keys = {} + cls.signer_principals = { + "llama.cpp": "dsv41-test-candidate", + "ds4": "dsv41-test-oracle", + } + cls.test_signers = {} + for runtime, principal in cls.signer_principals.items(): + signing_key = Path(cls._signing_directory.name) / f"{runtime.replace('.', '-')}-key" + subprocess.run( + [ + str(cls.ssh_keygen), + "-q", + "-t", "ed25519", + "-N", "", + "-f", str(signing_key), + ], + check=True, + ) + signing_key.chmod(0o600) + public_key = subprocess.check_output( + [str(cls.ssh_keygen), "-y", "-f", str(signing_key)], + text=True, + ).strip() + public_key = " ".join(public_key.split()[:2]) + lane = trace.ORACLE_LANE if runtime == "ds4" else trace.CANDIDATE_LANE + profile = "apple-metal" if runtime == "ds4" else "sibling-lib" + cls.signing_keys[runtime] = signing_key + cls.test_signers[principal] = { + "public_key": public_key, + "lane": lane, + "runtime": runtime, + "runtime_profile": profile, + } + cls.signing_key = cls.signing_keys["llama.cpp"] + cls.signer_principal = cls.signer_principals["llama.cpp"] + cls.verifier = cls._verifier_for_runtime("llama.cpp") + cls._trace_bundle_class = trace.TraceBundle + + @classmethod + def _verifier_for_runtime( + cls, + runtime: str, + *, + manifest_record: dict[str, object] | None = None, + expected_challenge: str = TEST_CHALLENGE, + expected_run_id: str | None = None, + verification_unix: int | None = None, + seen_run_ids: set[str] | None = None) -> trace.TraceVerifier: + principal = cls.signer_principals[runtime] + policy = cls.test_signers[principal] + manifest_record = manifest_record or manifest(runtime) + prompt_policy = fixture_prompt_builder_policy(b"abc") + prompt_policy["prompts"][0].update({ + "corpus_name": manifest_record["prompt"]["corpus_name"], + "corpus_sha256": manifest_record["prompt"]["corpus_sha256"], + "context": manifest_record["config"]["context"], + "decode_steps": manifest_record["config"]["decode_steps"], + "target_tokens": manifest_record["prompt"]["target_tokens"], + "prompt_sha256": manifest_record["prompt"]["sha256"], + "prompt_byte_count": manifest_record["prompt"]["byte_count"], + }) + prompt_policies = {TEST_PROMPT_BUILDER_POLICY_ID: prompt_policy} + candidate_policies = {} + ds4_policies = {} + candidate_policy_id = None + ds4_policy_id = None + if runtime == "llama.cpp": + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": manifest_record["candidate"]["revision"], + "profile": manifest_record["build"]["runtime_profile"]["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in manifest_record["build"]["runtime_libraries"] + ], + key=lambda item: item["component"], + ), + } + candidate_policy = { + "runtime": "llama.cpp", + "repository": manifest_record["candidate"]["repository"], + "revision": manifest_record["candidate"]["revision"], + "base_revision": manifest_record["candidate"]["base_revision"], + "diff_sha256": manifest_record["candidate"]["diff_sha256"], + "install_root": "/home/repo/build", + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "executable_path": manifest_record["candidate"]["executable_path"], + "executable_sha256": manifest_record["candidate"]["executable_sha256"], + "containment_helper": fixture_containment_helper( + manifest_record["candidate"]["revision"]), + "runtime_profile": copy.deepcopy(manifest_record["build"]["runtime_profile"]), + "runtime_receipt": receipt, + } + candidate_policies[TEST_CANDIDATE_EXPORTER_POLICY_ID] = candidate_policy + candidate_policy_id = TEST_CANDIDATE_EXPORTER_POLICY_ID + else: + ds4_policy = copy.deepcopy(DS4_EXPORTER_POLICY) + ds4_policies[TEST_DS4_EXPORTER_POLICY_ID] = ds4_policy + ds4_policy_id = TEST_DS4_EXPORTER_POLICY_ID + return trace.TraceVerifier.for_tests( + principal, + policy["public_key"], + lane=policy["lane"], + runtime=runtime, + runtime_profile=policy["runtime_profile"], + expected_challenge=expected_challenge, + expected_run_id=expected_run_id or TEST_RUN_IDS[runtime], + candidate_exporter_policies=candidate_policies, + ds4_exporter_policies=ds4_policies, + prompt_builder_policies=prompt_policies, + expected_candidate_exporter_policy_id=candidate_policy_id, + expected_ds4_exporter_policy_id=ds4_policy_id, + expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, + verification_unix=verification_unix or int(time.time()), + ssh_keygen=cls.ssh_keygen, + seen_run_ids=seen_run_ids, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls._signing_directory.cleanup() + + def setUp(self) -> None: + self._require_nvme_path = preflight.require_nvme_path + preflight.require_nvme_path = lambda path, label, **kwargs: preflight.resolved(path) + self._trace_bundle_symbol = trace.TraceBundle + + def test_bundle(root: Path, verify_blobs: bool = True, **_kwargs: object) -> object: + self._prune_fixture_extras(Path(root)) + signature = Path(root) / trace.SIGNATURE_NAME + if signature.exists() or signature.is_symlink(): + signature.unlink() + manifest_record = trace.strict_json_loads( + (Path(root) / trace.MANIFEST_NAME).read_text(encoding="ascii")) + runtime = manifest_record["runtime"] + principal = self.signer_principals[runtime] + authorization = manifest_record["authorization"] + verifier = self._verifier_for_runtime( + runtime, + manifest_record=manifest_record, + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + ) + trace.seal_bundle( + Path(root), + private_key=self.signing_keys[runtime], + principal=principal, + expected_lane=authorization["lane"], + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + candidate_exporter_policies=verifier.candidate_exporter_policies, + ds4_exporter_policies=verifier.ds4_exporter_policies, + prompt_builder_policies=verifier.prompt_builder_policies, + expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=verifier.expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, + expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, + expected_verifier_revision=verifier.expected_verifier_revision, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + return self._trace_bundle_class( + Path(root), + verify_blobs, + verifier=verifier, + ) + + trace.TraceBundle = test_bundle + + def tearDown(self) -> None: + preflight.require_nvme_path = self._require_nvme_path + trace.TraceBundle = self._trace_bundle_symbol + + def _seal_test_bundle(self, root: Path) -> str: + signature = root / trace.SIGNATURE_NAME + if signature.exists() or signature.is_symlink(): + signature.unlink() + manifest_record = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + runtime = manifest_record["runtime"] + principal = self.signer_principals[runtime] + authorization = manifest_record["authorization"] + verifier = self._verifier_for_runtime( + runtime, + manifest_record=manifest_record, + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + ) + return trace.seal_bundle( + root, + private_key=self.signing_keys[runtime], + principal=principal, + expected_lane=authorization["lane"], + expected_challenge=authorization["challenge"], + expected_run_id=authorization["run_id"], + candidate_exporter_policies=verifier.candidate_exporter_policies, + ds4_exporter_policies=verifier.ds4_exporter_policies, + prompt_builder_policies=verifier.prompt_builder_policies, + expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=verifier.expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, + expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, + expected_verifier_revision=verifier.expected_verifier_revision, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + def _prune_fixture_extras(self, root: Path) -> None: + manifest_path = root / trace.MANIFEST_NAME + events_path = root / trace.EVENTS_NAME + if not manifest_path.is_file() or not events_path.is_file(): + return + try: + manifest_record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + events = [ + trace.strict_json_loads(line) + for line in events_path.read_text(encoding="ascii").splitlines() + ] + except (OSError, UnicodeError, trace.TraceError): + return + expected = {trace.MANIFEST_NAME, trace.EVENTS_NAME} + if isinstance(manifest_record, dict): + prompt = manifest_record.get("prompt") + provenance = prompt.get("provenance") if isinstance(prompt, dict) else None + if isinstance(provenance, dict) and isinstance(provenance.get("path"), str): + expected.add(provenance["path"]) + audits = manifest_record.get("audits") + if isinstance(audits, dict): + for phase in audits.values(): + if not isinstance(phase, dict): + continue + for reference in phase.values(): + if not isinstance(reference, dict) or not isinstance(reference.get("path"), str): + continue + expected.add(reference["path"]) + audit_path = root / reference["path"] + if not audit_path.is_file(): + continue + try: + audit = trace.strict_json_loads(audit_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, trace.TraceError): + continue + nested = audit.get("data", {}).get("audit") if isinstance(audit, dict) else None + if isinstance(nested, dict) and isinstance(nested.get("path"), str): + expected.add(nested["path"]) + for event in events: + if isinstance(event, dict) and isinstance(event.get("blob"), str): + expected.add(event["blob"]) + for path in root.rglob("*"): + if path.is_file() and path.relative_to(root).as_posix() not in expected | {trace.SIGNATURE_NAME}: + path.unlink() + + def _read_sealed_bundle(self, root: Path) -> object: + return self._trace_bundle_class(root, verifier=self.verifier) + + def test_seal_requires_external_trust_and_fixed_verifier(self) -> None: + self.assertEqual(trace.APPROVED_TRACE_SIGNERS, {}) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(root) + self._read_sealed_bundle(root) + with self.assertRaisesRegex( + trace.TraceError, + "external signer, lane, challenge, run ID, and prompt builder approval"): + self._trace_bundle_class(root) + with self.assertRaisesRegex(trace.TraceError, "not approved"): + self._trace_bundle_class( + root, + signer_principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + expected_candidate_exporter_policy_id=TEST_CANDIDATE_EXPORTER_POLICY_ID, + expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, + ) + candidate_policy = self.test_signers[self.signer_principal] + unknown = trace.TraceVerifier.for_tests( + "unknown", + candidate_policy["public_key"], + lane=trace.CANDIDATE_LANE, + runtime="llama.cpp", + runtime_profile="sibling-lib", + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + verification_unix=int(time.time()), + ) + with self.assertRaisesRegex(trace.TraceError, "externally expected signer"): + self._trace_bundle_class(root, verifier=unknown) + + fake_directory = Path(temp) / "fake-bin" + fake_directory.mkdir() + fake = fake_directory / "ssh-keygen" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") + fake.chmod(0o755) + with mock.patch.dict(os.environ, {"PATH": str(fake_directory)}): + self._read_sealed_bundle(root) + substituted = Path(temp) / "substituted-ssh-keygen" + substituted.symlink_to(fake) + verifier = trace.TraceVerifier.for_tests( + self.signer_principal, + candidate_policy["public_key"], + lane=trace.CANDIDATE_LANE, + runtime="llama.cpp", + runtime_profile="sibling-lib", + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + verification_unix=int(time.time()), + ssh_keygen=substituted, + ) + with self.assertRaisesRegex(trace.TraceError, "non-symlink"): + self._trace_bundle_class(root, verifier=verifier) + + with self.assertRaisesRegex(trace.TraceError, "already exists"): + trace.seal_bundle( + root, + private_key=self.signing_key, + principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + other_key = Path(temp) / "other-key" + subprocess.run( + [ + str(self.ssh_keygen), + "-q", + "-t", "ed25519", + "-N", "", + "-f", str(other_key), + ], + check=True, + ) + other_key.chmod(0o600) + with self.assertRaisesRegex(trace.TraceError, "does not match"): + trace.validate_signing_identity( + other_key, + self.signer_principal, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + def test_production_executable_approval_maps_fail_closed(self) -> None: + self.assertEqual(trace.APPROVED_CANDIDATE_EXPORTERS, {}) + self.assertEqual(trace.APPROVED_DS4_EXPORTERS, {}) + self.assertEqual(trace.APPROVED_PROMPT_BUILDERS, {}) + self.assertEqual(trace.APPROVED_EXECUTABLE_APPROVERS, {}) + with self.assertRaisesRegex(trace.TraceError, "candidate exporter approval is not trusted"): + trace.candidate_exporter_approval(TEST_CANDIDATE_EXPORTER_POLICY_ID) + with self.assertRaisesRegex(trace.TraceError, "ds4 exporter approval is not trusted"): + trace.ds4_exporter_approval(TEST_DS4_EXPORTER_POLICY_ID) + with self.assertRaisesRegex(trace.TraceError, "prompt builder approval is not trusted"): + trace.prompt_builder_approval(TEST_PROMPT_BUILDER_POLICY_ID) + + def test_external_executable_approval_signature_and_tamper(self) -> None: + verifier = self._verifier_for_runtime("llama.cpp") + principal = "dsv41-test-executable-approver" + public_key = self.test_signers[self.signer_principal]["public_key"] + policy = { + "format": trace.EXECUTABLE_APPROVAL_FORMAT, + "version": trace.EXECUTABLE_APPROVAL_VERSION, + "principal": principal, + "verifier_repository": trace.REPOSITORY, + "verifier_revision": "a" * 40, + "candidate_exporters": verifier.candidate_exporter_policies, + "ds4_exporters": verifier.ds4_exporter_policies, + "prompt_builders": verifier.prompt_builder_policies, + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy_path = root / "approval.json" + policy_path.write_text(trace.canonical_json(policy) + "\n", encoding="ascii") + subprocess.run( + [ + str(self.ssh_keygen), + "-Y", "sign", + "-f", str(self.signing_key), + "-n", trace.EXECUTABLE_APPROVAL_NAMESPACE, + str(policy_path), + ], + check=True, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + signature_path = policy_path.with_suffix(".json.sig") + approvers = { + principal: { + "public_key": public_key, + "policy_root": str(root), + "owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + }, + } + loaded = trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers=approvers, + ssh_keygen=self.ssh_keygen, + test_only_trust=True, + ) + self.assertEqual(loaded.verifier_revision, "a" * 40) + self.assertEqual(loaded.sha256, trace.sha256_file(policy_path)) + self.assertEqual( + loaded.candidate_exporters, + verifier.candidate_exporter_policies, + ) + self.assertEqual(loaded.ds4_exporters, verifier.ds4_exporter_policies) + with self.assertRaisesRegex(trace.TraceError, "outside protected output roots"): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers=approvers, + ssh_keygen=self.ssh_keygen, + forbidden_roots=(root,), + test_only_trust=True, + ) + tampered = copy.deepcopy(policy) + tampered["verifier_revision"] = "b" * 40 + policy_path.write_text(trace.canonical_json(tampered) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "signature verification failed"): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers=approvers, + ssh_keygen=self.ssh_keygen, + test_only_trust=True, + ) + with self.assertRaisesRegex(trace.TraceError, "principal is not trusted"): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers={}, + ssh_keygen=self.ssh_keygen, + test_only_trust=True, + ) + + def test_external_approval_rejects_mutable_root_and_hardlinks(self) -> None: + verifier = self._verifier_for_runtime("llama.cpp") + principal = "dsv41-test-executable-approver" + policy = { + "format": trace.EXECUTABLE_APPROVAL_FORMAT, + "version": trace.EXECUTABLE_APPROVAL_VERSION, + "principal": principal, + "verifier_repository": trace.REPOSITORY, + "verifier_revision": "a" * 40, + "candidate_exporters": verifier.candidate_exporter_policies, + "ds4_exporters": verifier.ds4_exporter_policies, + "prompt_builders": verifier.prompt_builder_policies, + } + for mutation, message in ( + ("mutable-root", "path is mutable"), + ("hard-linked-policy", "one-link regular file"), + ("hard-linked-signature", "one-link regular file"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy_path = root / "approval.json" + policy_path.write_text(trace.canonical_json(policy) + "\n", encoding="ascii") + subprocess.run( + [ + str(self.ssh_keygen), + "-Y", "sign", + "-f", str(self.signing_key), + "-n", trace.EXECUTABLE_APPROVAL_NAMESPACE, + str(policy_path), + ], + check=True, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + signature_path = policy_path.with_suffix(".json.sig") + policy_path.chmod(0o444) + signature_path.chmod(0o444) + root.chmod(0o555) + if mutation == "mutable-root": + root.chmod(0o777) + elif mutation == "hard-linked-policy": + os.link(policy_path, Path(temp).parent / f"{root.name}-policy-alias") + else: + os.link(signature_path, Path(temp).parent / f"{root.name}-signature-alias") + approvers = { + principal: { + "public_key": self.test_signers[self.signer_principal]["public_key"], + "policy_root": str(root), + "owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + }, + } + try: + with isolated_test_install_trust(), self.assertRaisesRegex(trace.TraceError, message): + trace.load_executable_approval_policy( + policy_path, + signature_path, + expected_principal=principal, + trusted_approvers=approvers, + ssh_keygen=self.ssh_keygen, + ) + finally: + root.chmod(0o755) + for alias in Path(temp).parent.glob(f"{root.name}-*-alias"): + alias.unlink() + + def test_external_ds4_approval_requires_distinct_producer_and_verifier(self) -> None: + principal = "dsv41-test-executable-approver" + policy = { + "format": trace.EXECUTABLE_APPROVAL_FORMAT, + "version": trace.EXECUTABLE_APPROVAL_VERSION, + "principal": principal, + "verifier_repository": trace.REPOSITORY, + "verifier_revision": trace.DS4_REVISION, + "candidate_exporters": {}, + "ds4_exporters": { + TEST_DS4_EXPORTER_POLICY_ID: copy.deepcopy(DS4_EXPORTER_POLICY), + }, + "prompt_builders": {}, + } + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy_path = root / "approval.json" + policy_path.write_text(trace.canonical_json(policy) + "\n", encoding="ascii") + subprocess.run( + [ + str(self.ssh_keygen), + "-Y", "sign", + "-f", str(self.signing_key), + "-n", trace.EXECUTABLE_APPROVAL_NAMESPACE, + str(policy_path), + ], + check=True, + stdin=subprocess.DEVNULL, + capture_output=True, + ) + with self.assertRaisesRegex(trace.TraceError, "producer revision must differ"): + trace.load_executable_approval_policy( + policy_path, + policy_path.with_suffix(".json.sig"), + expected_principal=principal, + trusted_approvers={ + principal: { + "public_key": self.test_signers[self.signer_principal]["public_key"], + "policy_root": str(root), + "owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + }, + }, + ssh_keygen=self.ssh_keygen, + test_only_trust=True, + ) + + def test_ds4_approval_rejects_runtime_identity_mismatches(self) -> None: + for mutation, message in ( + (lambda value: value.update({"runtime": "llama.cpp"}), "runtime identity"), + (lambda value: value.update({"repository": trace.REPOSITORY}), "runtime identity"), + (lambda value: value.update({"revision": "a" * 40}), "runtime identity"), + ( + lambda value: value.update({ + "executable_path": "/Users/oracle/other/bin/ds4-trace", + }), + "outside its install policy", + ), + ( + lambda value: value["runtime_profile"].update({ + "selected_backend_component": "missing", + }), + "runtime profile", + ), + ( + lambda value: value["runtime_receipt"].update({"profile": "co-located"}), + "runtime receipt identity", + ), + ( + lambda value: [ + component.update({"revision": None}) + for component in value["runtime_receipt"]["components"] + ], + "receipt differs|pinned ds4 revision", + ), + ): + with self.subTest(message=message): + policy = copy.deepcopy(DS4_EXPORTER_POLICY) + mutation(policy) + with self.assertRaisesRegex(trace.TraceError, message): + trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + ) + + def test_candidate_runner_rejects_unapproved_exporter_before_execution(self) -> None: + argv = [ + "run_llama.py", + "--exporter", "/usr/bin/true", + "--repo", "/tmp/repo", + "--candidate-revision", "a" * 40, + "--base-revision", "b" * 40, + "--candidate-diff-sha256", "c" * 64, + "--candidate-exporter-policy-id", "unapproved-exporter", + "--prompt-builder-policy-id", "unapproved-builder", + "--approval-policy", "/tmp/approval.json", + "--approval-signature", "/tmp/approval.sig", + "--approval-principal", "unapproved", + "--corpus-name", "correctness-prose.txt", + "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], + "--prompt-provenance", "/tmp/prompt.json", + "--model", "/tmp/model.gguf", + "--prompt", "/tmp/prompt.txt", + "--output", "/tmp/output", + "--signer-principal", "candidate", + "--signing-key", "/tmp/key", + "--execution-challenge", TEST_CHALLENGE, + "--run-id", TEST_RUN_IDS["llama.cpp"], + "--authorization-issued-unix", str(TEST_AUTH_ISSUED), + "--authorization-expires-unix", str(TEST_AUTH_EXPIRES), + "--preflight-only", + ] + with mock.patch.dict(os.environ, {}, clear=True), mock.patch.object( + sys, "argv", argv), mock.patch.object( + sys, "stderr", io.StringIO()), mock.patch.object( + run_llama, "query_runtime_build_attestation") as build_query, mock.patch.object( + run_llama, "query_accelerator_attestation") as accelerator_query: + self.assertEqual(run_llama.main(), 1) + build_query.assert_not_called() + accelerator_query.assert_not_called() + + def test_prompt_builder_rejects_unapproved_identity_before_execution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "actual" / "bin" / "llama-deepseek-v41-prompt-builder" + approved_builder = root / "approved" / "bin" / "llama-deepseek-v41-prompt-builder" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + policy = fixture_prompt_builder_policy( + b"prompt", + builder_path=str(approved_builder), + builder_sha256=trace.sha256_file(builder), + source_root=str(Path(__file__).parents[1].resolve()), + ) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + with mock.patch.object(run_matrix.subprocess, "run") as execute, self.assertRaisesRegex( + run_matrix.TraceError, "path differs from external approval"): + run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=policy, + builder_policy_sha256=policy_sha256, + model=root / "model.gguf", + corpus=root / "corpus.txt", + source_corpus=Path(__file__).parents[1] / "tests" / "corpus" / "correctness-prose.txt", + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=root / "prompt.txt", + context=3, + decode_steps=1, + ) + execute.assert_not_called() + + def test_approved_executable_uses_linux_descriptor_path(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + helper = install / "bin" / "llama-deepseek-v41-containment-helper" + library = install / "lib" / "libapproved.so" + executable.parent.mkdir(parents=True) + library.parent.mkdir() + executable.write_bytes(b"approved") + helper.write_bytes(b"helper") + library.write_bytes(b"approved library") + executable.chmod(0o555) + helper.chmod(0o555) + library.chmod(0o555) + policy = { + "revision": "a" * 40, + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "containment_helper": fixture_containment_helper( + "a" * 40, trace.sha256_file(helper)), + "runtime_receipt": { + "components": [{ + "component": "approved", + "filename": library.name, + "sha256": trace.sha256_file(library), + }], + }, + } + completed = subprocess.CompletedProcess([str(executable)], 0, b"", b"") + contained = trace._ContainedRun(completed, None, [], None, True, True) + with isolated_test_install_trust(), mock.patch.object( + trace.sys, "platform", "linux"), mock.patch.object( + trace, "_run_contained_process", return_value=contained) as execute: + result, identity = trace.run_approved_executable( + [str(executable), "--version"], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.stdout, "") + self.assertEqual(identity.path, str(executable)) + launch = execute.call_args.kwargs["launch"] + self.assertRegex(launch["executable"], r"^/proc/self/fd/[0-9]+$") + self.assertEqual(execute.call_args.args[0][0], str(executable)) + self.assertEqual(len(launch["pass_fds"]), 2) + + def test_approved_executable_postchecks_before_strict_decode(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"approved") + executable.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": {"components": []}, + } + events = [] + completed = subprocess.CompletedProcess([str(executable)], 0, b"\xff", b"") + contained = trace._ContainedRun(completed, None, [], None, True, True) + original_verify = trace.verify_approved_executable_identity + original_decode = trace._decode_subprocess_stream + + def verify(*args: object, **kwargs: object) -> None: + events.append("verify") + original_verify(*args, **kwargs) + + def decode(*args: object, **kwargs: object) -> bytes | str | None: + events.append("decode") + return original_decode(*args, **kwargs) + + with isolated_test_install_trust(), mock.patch.object( + trace, "_run_contained_process", return_value=contained), mock.patch.object( + trace, "verify_approved_executable_identity", side_effect=verify), mock.patch.object( + trace, "_decode_subprocess_stream", side_effect=decode), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace.run_approved_executable( + [str(executable)], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + check=False, + capture_output=True, + text=True, + ) + self.assertIsInstance(raised.exception.__cause__, UnicodeDecodeError) + self.assertTrue(raised.exception.quiescence_proven) + self.assertGreater(events.count("verify"), 1) + self.assertLess(max(index for index, event in enumerate(events) if event == "verify"), events.index("decode")) + + def test_approved_install_rejects_mutable_alias_and_hardlink_paths(self) -> None: + for mutation, message in ( + ("writable-root", "path is mutable"), + ("writable-ancestor", "path is mutable"), + ("writable-executable", "immutable trusted-owned"), + ("hard-linked-library", "immutable trusted-owned"), + ("symlinked-executable", "canonical|aliases"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + install = root / "install" + executable = install / "bin" / "approved" + library = install / "lib" / "libapproved.so" + executable.parent.mkdir(parents=True) + library.parent.mkdir() + executable.write_bytes(b"approved executable") + library.write_bytes(b"approved library") + alias = install / "bin" / "alias" + if mutation == "symlinked-executable": + alias.symlink_to(executable) + executable.chmod(0o555) + library.chmod(0o555) + executable.parent.chmod(0o555) + library.parent.chmod(0o555) + install.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": { + "components": [{ + "component": "approved", + "filename": library.name, + "sha256": trace.sha256_file(library), + }], + }, + } + path = executable + if mutation == "writable-root": + install.chmod(0o777) + elif mutation == "writable-ancestor": + root.chmod(0o777) + elif mutation == "writable-executable": + executable.chmod(0o755) + elif mutation == "hard-linked-library": + os.link(library, root / "library-alias") + else: + path = alias + with isolated_test_install_trust(), self.assertRaisesRegex(trace.TraceError, message): + if mutation == "hard-linked-library": + trace.approved_runtime_file_identities(policy, label="approved") + else: + trace.approved_executable_identity( + path, + install_root=str(install), + expected_owner_uid=policy["install_owner_uid"], + expected_path=str(path), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + ) + + def test_writable_install_root_blocks_replace_restore_before_launch(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"approved executable") + executable.chmod(0o555) + install.chmod(0o777) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": {"components": []}, + } + with isolated_test_install_trust(), mock.patch.object( + trace.sys, "platform", "linux"), mock.patch.object( + trace, "_run_contained_process") as execute, self.assertRaisesRegex( + trace.TraceError, "path is mutable"): + trace.run_approved_executable( + [str(executable)], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + ) + execute.assert_not_called() + + def test_ds4_exporter_revalidates_each_invocation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + first = run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + check=True, + capture_output=True, + ) + self.assertEqual(first.stdout, b"approved\n") + exporter.parent.chmod(0o755) + exporter.chmod(0o755) + exporter.write_text("#!/bin/sh\nprintf 'replacement\\n'\n", encoding="ascii") + exporter.chmod(0o555) + exporter.parent.chmod(0o555) + with mock.patch.object(trace, "_run_contained_process") as execute, self.assertRaisesRegex( + run_ds4.TraceError, "SHA-256 differs|identity changed"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + check=True, + capture_output=True, + ) + execute.assert_not_called() + + def test_ds4_exporter_detects_swap_after_precheck(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + backup = root / "approved-backup" + replacement = root / "replacement" + replacement.write_text("#!/bin/sh\nprintf 'replacement\\n'\n", encoding="ascii") + replacement.chmod(0o555) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + + def swap_after_precheck(*_args: object, **_kwargs: object) -> trace._ContainedRun: + exporter.parent.chmod(0o755) + exporter.rename(backup) + replacement.rename(exporter) + exporter.parent.chmod(0o555) + result = subprocess.CompletedProcess([str(exporter)], 0, b"replacement\n", b"") + return trace._ContainedRun(result, None, [], None, True, True) + + with mock.patch.object( + sys.modules["trace_format"], + "_run_contained_process", + side_effect=swap_after_precheck, + ), self.assertRaisesRegex( + run_ds4.TraceError, "descriptor identity changed|SHA-256 differs|identity changed"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + check=True, + capture_output=True, + ) + + def test_ds4_exporter_postchecks_failed_invocation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + backup = root / "approved-backup" + replacement = root / "replacement" + replacement.write_text("#!/bin/sh\nexit 7\n", encoding="ascii") + replacement.chmod(0o555) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + + def fail_after_swap(*_args: object, **_kwargs: object) -> trace._ContainedRun: + exporter.parent.chmod(0o755) + exporter.rename(backup) + replacement.rename(exporter) + exporter.parent.chmod(0o555) + error = subprocess.TimeoutExpired([str(exporter)], 7) + return trace._ContainedRun(None, error, [], None, True, True) + + with mock.patch.object( + sys.modules["trace_format"], + "_run_contained_process", + side_effect=fail_after_swap, + ), self.assertRaisesRegex( + run_ds4.TraceError, + "primary failure \\[TimeoutExpired:.*secondary integrity failures:.*" + "executable-path-root.*(SHA-256 differs|identity changed)"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + check=True, + capture_output=True, + ) + + def test_ds4_exporter_aggregates_all_postcheck_failures(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + runtime_path = ( + Path(policy["install_root"]) / "lib" / + policy["runtime_receipt"]["components"][0]["filename"]) + primary = subprocess.TimeoutExpired([str(exporter)], 7) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + + def fail_and_mutate(*_args: object, **_kwargs: object) -> trace._ContainedRun: + exporter.chmod(0o755) + exporter.write_text("#!/bin/sh\nexit 9\n", encoding="ascii") + runtime_path.chmod(0o755) + runtime_path.write_bytes(b"mutated runtime") + Path(policy["install_root"]).chmod(0o777) + cleanup = trace._IntegrityFailure( + "process-tree-quiescence", trace.TraceError("cleanup deadline expired")) + return trace._ContainedRun(None, primary, [cleanup], None, False, True) + + with mock.patch.object( + sys.modules["trace_format"], + "_run_contained_process", + side_effect=fail_and_mutate, + ), self.assertRaises(sys.modules["trace_format"].ExecutionIntegrityError) as raised: + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + check=False, + capture_output=True, + ) + error = raised.exception + self.assertIs(error.__cause__, primary) + self.assertIs(error.primary_error, primary) + components = {failure.component for failure in error.secondary_errors} + self.assertIn("process-tree-quiescence", components) + self.assertIn("executable-descriptor", components) + self.assertIn("executable-path-root", components) + self.assertTrue(any(item.startswith("runtime-descriptor:") for item in components)) + self.assertTrue(any(item.startswith("runtime-path-root:") for item in components)) + + @unittest.skipUnless(os.name == "posix", "POSIX containment test") + def test_approved_executable_timeout_contains_setsid_descendant(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + marker = root / "descendant-survived" + exporter.chmod(0o755) + if sys.platform == "linux": + exporter.write_text( + f"#!{sys.executable}\n" + "import os\n" + "import signal\n" + "import sys\n" + "import time\n" + "if os.fork() == 0:\n" + " os.setsid()\n" + " signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + " time.sleep(1)\n" + " open(sys.argv[1], 'w', encoding='ascii').write('survived')\n" + " os._exit(0)\n" + "time.sleep(30)\n", + encoding="ascii", + ) + else: + exporter.write_text( + "#!/bin/sh\n" + "( sleep 1; printf survived > \"$1\" ) /dev/null 2>&1 &\n" + "sleep 30\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + with isolated_test_install_trust(), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace.run_approved_executable( + [str(exporter), str(marker)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="approved executable", + timeout=0.1, + check=False, + capture_output=True, + ) + self.assertIsInstance(raised.exception.__cause__, subprocess.TimeoutExpired) + time.sleep(1.2) + self.assertFalse(marker.exists()) + + def test_linux_native_helper_signals_only_stable_pidfds(self) -> None: + containment = trace._ProcessContainment( + process=mock.Mock(pid=77), + linux_root_pidfd=90, + linux_namespace_pidfd=91, + linux_lock_held=True, + linux_exec_released=True, + ) + with mock.patch.object(trace, "_linux_signal_pidfd") as signal_pidfd, mock.patch.object( + trace.os, "kill") as numeric_kill: + failures = trace._linux_signal_owned_children(containment, trace.signal.SIGKILL) + self.assertEqual(failures, []) + self.assertEqual( + signal_pidfd.call_args_list, + [mock.call(91, trace.signal.SIGKILL), mock.call(90, trace.signal.SIGKILL)], + ) + numeric_kill.assert_not_called() + + def test_linux_native_helper_boundary_precedes_target_release(self) -> None: + start_source = inspect.getsource(trace._start_linux_native_helper) + spawn_source = inspect.getsource(trace._start_linux_native_helper_process) + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + helper_main = helper_source[helper_source.index("int run_linux_helper"):] + self.assertNotIn("os.fork", start_source + spawn_source) + self.assertIn("os.posix_spawn", spawn_source) + self.assertLess(spawn_source.index("os.posix_spawn"), spawn_source.index("_linux_open_pidfd")) + self.assertLess(start_source.index("process.root_pidfd"), start_source.index("process.release_exec")) + self.assertIn("PR_SET_PDEATHSIG", helper_source) + for flag in ("CLONE_NEWUSER", "CLONE_NEWPID", "CLONE_NEWNS", "CLONE_PIDFD"): + self.assertIn(flag, helper_source) + self.assertLess( + helper_main.index("namespace_owner owned_namespace"), + helper_main.index('"uid_map"'), + ) + self.assertIn('"uid_map"', helper_source) + self.assertIn('mount("proc", "/proc", "proc"', helper_source) + self.assertLess( + helper_source.index("target PID namespace did not bind helper lifetime"), + helper_source.index('send_descriptor(config.protocol_fd, \"PREPARED\"'), + ) + self.assertLess( + helper_source.index('send_descriptor(config.protocol_fd, \"PREPARED\"'), + helper_source.index('!= \"EXEC\"'), + ) + + def test_linux_native_helper_requires_zero_group_service(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + run_source = helper_source[ + helper_source.index("int run_linux_helper"): + helper_source.index("#endif\n\n}") + ] + init_source = helper_source[ + helper_source.index("[[noreturn]] void run_namespace_init"): + helper_source.index("int run_linux_helper") + ] + target_source = helper_source[ + helper_source.index("[[noreturn]] void run_target_bootstrap"): + helper_source.index("[[noreturn]] void run_namespace_init") + ] + self.assertIn("--check-launcher-groups", helper_source) + self.assertIn("supplementary-groups=0", helper_source) + self.assertIn( + 'require_zero_supplementary_groups("containment launcher");', + run_source, + ) + self.assertLess( + run_source.index('require_zero_supplementary_groups("containment launcher");'), + run_source.index("require_initial_signal_state();"), + ) + self.assertLess( + run_source.index("require_initial_signal_state();"), + run_source.index("CLONE_NEWUSER"), + ) + self.assertNotIn("setgroups(", helper_source) + self.assertLess( + init_source.index('stage = "namespace-groups-verify"'), + init_source.index('stage = "namespace-setresgid"'), + ) + self.assertLess( + target_source.index('stage = "target-groups-verify"'), + target_source.index('stage = "target-privilege-drop"'), + ) + parent_source = inspect.getsource(trace._start_linux_native_helper) + self.assertLess( + parent_source.index("_require_zero_supplementary_groups()"), + parent_source.index("_start_linux_native_helper_process"), + ) + + def test_linux_parent_requires_zero_group_service_before_spawn(self) -> None: + with mock.patch.object(trace.os, "getgroups", return_value=[]): + trace._require_zero_supplementary_groups() + with mock.patch.object(trace.os, "getgroups", return_value=[10, 39, 105]), ( + self.assertRaisesRegex( + trace.TraceError, + "requires zero supplementary groups; found 3")): + trace._require_zero_supplementary_groups() + query_error = PermissionError(1, "Operation not permitted") + with mock.patch.object(trace.os, "getgroups", side_effect=query_error), ( + self.assertRaisesRegex( + trace.TraceError, + "cannot query Linux containment launcher supplementary groups")): + trace._require_zero_supplementary_groups() + + def test_posix_spawn_does_not_run_registered_atfork_callback(self) -> None: + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp) / "atfork-ran" + probe = ( + "import os,sys\n" + "marker=sys.argv[1]\n" + "os.register_at_fork(after_in_child=lambda: open(marker,'w',encoding='ascii').write('ran'))\n" + "pid=os.posix_spawn('/usr/bin/true',['/usr/bin/true'],os.environ)\n" + "os.waitpid(pid,0)\n" + "raise SystemExit(1 if os.path.exists(marker) else 0)\n" + ) + subprocess.run( + [sys.executable, "-c", probe, str(marker)], + check=True, + env={"PATH": os.environ.get("PATH", "")}, + ) + self.assertFalse(marker.exists()) + + def test_linux_native_helper_inherited_signal_defenses_are_explicit(self) -> None: + spawn_source = inspect.getsource(trace._start_linux_native_helper_process) + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + self.assertIn("setsigmask=_all_catchable_signals()", spawn_source) + self.assertIn("setsigdef=_all_catchable_signals()", spawn_source) + self.assertIn("setsid=True", spawn_source) + self.assertIn("require_initial_signal_state();", helper_source) + self.assertLess( + helper_source.index("require_initial_signal_state();"), + helper_source.index('send_packet(config.protocol_fd, \"READY\")'), + ) + + def test_linux_native_helper_source_isolates_target_privileges_before_exec(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + bootstrap_start = helper_source.index("[[noreturn]] void run_target_bootstrap") + init_start = helper_source.index("[[noreturn]] void run_namespace_init") + bootstrap_source = helper_source[bootstrap_start:init_start] + init_source = helper_source[init_start:helper_source.index("int run_linux_helper")] + self.assertLess( + bootstrap_source.index("set_parent_death(1);"), + bootstrap_source.index("make_isolated_session();"), + ) + self.assertLess( + bootstrap_source.index("make_isolated_session();"), + bootstrap_source.index("drop_target_privileges();"), + ) + self.assertLess( + bootstrap_source.index("drop_target_privileges();"), + bootstrap_source.index("execve("), + ) + self.assertLess( + bootstrap_source.index('send_descriptor(security_fd, \"FILTER\", listener);'), + bootstrap_source.index("verify_target_attack_denials();"), + ) + self.assertLess( + bootstrap_source.index("verify_target_attack_denials();"), + bootstrap_source.index('send_packet(security_fd, \"VERIFIED\");'), + ) + self.assertLess( + bootstrap_source.index('receive_packet(security_fd) != \"GO\"'), + bootstrap_source.index("execve("), + ) + for required in ( + "PR_SET_DUMPABLE", + "PR_SET_PTRACER", + "PR_SET_SECUREBITS", + "SECBIT_NOROOT_LOCKED", + "SECBIT_NO_SETUID_FIXUP_LOCKED", + "SECBIT_KEEP_CAPS_LOCKED", + "SECBIT_NO_CAP_AMBIENT_RAISE_LOCKED", + "PR_CAPBSET_DROP", + "PR_CAP_AMBIENT_CLEAR_ALL", + "SYS_capset", + "PR_SET_NO_NEW_PRIVS", + "SECCOMP_RET_USER_NOTIF", + "SECCOMP_FILTER_FLAG_NEW_LISTENER", + "SECCOMP_IOCTL_NOTIF_RECV", + "target attempted a forbidden lifecycle operation", + "__NR_ptrace", + "__NR_process_vm_readv", + "__NR_process_vm_writev", + "__NR_kill", + "__NR_setresuid", + "__NR_setpgid", + "__NR_setsid"): + self.assertIn(required, helper_source) + self.assertIn("target_arguments.flags = CLONE_NEWUSER | CLONE_PIDFD;", helper_source) + self.assertIn('"65534 0 1\\n"', helper_source) + self.assertLess( + helper_source.index("protect_namespace_init();"), + helper_source.index('write_all(ready_fd, \"R\", 1);'), + ) + self.assertLess( + init_source.index('target_ready != \'I\''), + init_source.index('write_all(ready_fd, \"I\", 1);'), + ) + self.assertLess( + init_source.index('receive_descriptor(target_security[0], \"FILTER\")'), + init_source.index("verify_target_isolation_probes(target_listener);"), + ) + self.assertLess( + init_source.index("verify_target_isolation_probes(target_listener);"), + init_source.index('send_packet(target_security[0], \"GO\");'), + ) + self.assertLess( + init_source.index('write_all(ready_fd, \"I\", 1);'), + init_source.index("wait_for_isolated_target(owned_target, target_listener)"), + ) + self.assertLess( + init_source.index('write_all(ready_fd, \"C\", 1);'), + init_source.index("_exit(wait_status_exit_code(target_status));"), + ) + helper_main = helper_source[helper_source.index("int run_linux_helper"):] + self.assertLess( + helper_main.index("owned_namespace.wait();"), + helper_main.index("target namespace teardown did not complete"), + ) + self.assertLess( + helper_main.index("target namespace teardown did not complete"), + helper_main.index('send_packet(config.protocol_fd, \"COMPLETE\");'), + ) + + def test_linux_native_helper_without_pidfd_support_fails_before_spawn(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support", + side_effect=trace.TraceError("pidfd unavailable")), mock.patch.object( + trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( + trace.TraceError, "pidfd unavailable"): + trace._start_linux_native_helper(["approved"], {}) + start.assert_not_called() + lock.release.assert_called_once() + + def test_linux_helper_pidfd_failure_aborts_before_protocol_release(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + primary = OSError("pidfd failed") + launch_error = trace.ExecutionIntegrityError( + "helper pidfd failed", + primary_error=primary, + secondary_errors=[], + quiescence_proven=False, + ) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_start_linux_native_helper_process", side_effect=launch_error), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper(["approved"], {}) + self.assertFalse(raised.exception.quiescence_proven) + self.assertIs(raised.exception.primary_error, primary) + lock.release.assert_called_once() + + def test_linux_helper_pidfd_open_failure_reaps_blocked_helper(self) -> None: + parent_socket = mock.Mock() + child_socket = mock.Mock() + parent_socket.fileno.return_value = 50 + child_socket.fileno.return_value = 51 + primary = OSError("pidfd failed") + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object( + trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( + trace.os, "get_inheritable", return_value=False), mock.patch.object( + trace.os, "set_inheritable"), mock.patch.object( + trace.os, "posix_spawn", return_value=71), mock.patch.object( + trace, "_linux_open_pidfd", side_effect=primary), mock.patch.object( + trace.os, "waitpid", return_value=(71, 0)), mock.patch.object( + trace.os, "kill") as numeric_kill, self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper( + ["/bin/true"], + { + "_containment_helper_path": "/approved/helper", + "_containment_helper_descriptor": 40, + }, + ) + self.assertIs(raised.exception.primary_error, primary) + self.assertFalse(raised.exception.quiescence_proven) + parent_socket.shutdown.assert_called_once_with(trace.socket.SHUT_RDWR) + parent_socket.close.assert_called_once() + child_socket.close.assert_called_once() + numeric_kill.assert_not_called() + + def test_linux_post_spawn_cleanup_preserves_all_failures_and_reaps(self) -> None: + parent_socket = mock.Mock() + child_socket = mock.Mock() + parent_socket.fileno.return_value = 50 + child_socket.fileno.return_value = 51 + child_socket.close.side_effect = OSError("child protocol close failed") + restore_error = OSError("inheritability restore failed") + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object( + trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace.socket, "socketpair", return_value=(parent_socket, child_socket)), mock.patch.object( + trace.os, "get_inheritable", return_value=False), mock.patch.object( + trace.os, "set_inheritable", side_effect=[None, restore_error]), mock.patch.object( + trace.os, "posix_spawn", return_value=71), mock.patch.object( + trace, "_linux_open_pidfd", return_value=90), mock.patch.object( + trace, "_linux_signal_pidfd") as signal_pidfd, mock.patch.object( + trace.os, "waitpid", return_value=(71, 0)), mock.patch.object( + trace.os, "close"), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper( + ["/bin/true"], + { + "_containment_helper_path": "/approved/helper", + "_containment_helper_descriptor": 40, + }, + ) + self.assertIs(raised.exception.primary_error, restore_error) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-helper-child-protocol-close"], + ) + self.assertFalse(raised.exception.quiescence_proven) + signal_pidfd.assert_not_called() + parent_socket.close.assert_called_once() + + def test_linux_failed_startup_reap_retains_locked_authority(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + primary = OSError("launch cleanup failed") + reap_failure = trace._IntegrityFailure( + "linux-blocked-child-reap", subprocess.TimeoutExpired(["helper"], 5)) + process = mock.Mock( + pid=71, + root_pidfd=90, + namespace_pidfd=None, + launch_primary_error=primary, + launch_integrity_failures=[], + ) + process.abort_blocked.return_value = trace._ContainmentCleanup( + [reap_failure], False) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace, "_LINUX_POISONED_CONTAINMENT", None), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( + trace.os, "close") as close: + with self.assertRaises(trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper(["approved"], {}) + poisoned = trace._LINUX_HELPER_POISONED + poisoned_containment = trace._LINUX_POISONED_CONTAINMENT + self.assertIs(raised.exception.primary_error, primary) + self.assertFalse(raised.exception.quiescence_proven) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-blocked-child-reap", "linux-helper-teardown"], + ) + self.assertTrue(poisoned) + self.assertIs(poisoned_containment.process, process) + process.close_streams.assert_not_called() + close.assert_not_called() + lock.release.assert_not_called() + + def test_linux_blocked_helper_reap_retries_after_protocol_close(self) -> None: + protocol = mock.Mock() + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + process, + "wait", + side_effect=[subprocess.TimeoutExpired(["approved"], 5), 0], + ) as wait: + cleanup = process.abort_blocked() + self.assertFalse(cleanup.quiescence_proven) + self.assertEqual( + [failure.component for failure in cleanup.failures], + ["linux-blocked-child-reap"], + ) + self.assertEqual(wait.call_count, 2) + protocol.shutdown.assert_called_once_with(trace.socket.SHUT_RDWR) + protocol.close.assert_called_once() + self.assertIsNone(process._protocol_socket) + + def test_linux_namespace_unavailable_never_sends_exec(self) -> None: + protocol = mock.Mock() + protocol.recvmsg.side_effect = [ + (b"READY", [], 0, None), + (b"PREPARED", [], 0, None), + ] + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with self.assertRaisesRegex(trace.TraceError, "did not provide one namespace pidfd"): + process.release_exec() + self.assertEqual(protocol.sendall.call_args_list, [mock.call(b"PREPARE")]) + self.assertIsNone(process.namespace_pidfd) + + def test_linux_protocol_accepts_echoed_cmsg_cloexec(self) -> None: + protocol = mock.Mock() + protocol.recvmsg.return_value = (b"READY", [], 1073741824, None) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True): + process._receive_protocol(b"READY") + protocol.recvmsg.assert_called_once_with( + 128, + trace.socket.CMSG_SPACE( + array.array("i").itemsize * + trace.LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS), + 1073741824, + ) + + def test_linux_protocol_accepts_one_cloexec_pidfd(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace, "_linux_fd_is_close_on_exec", return_value=True) as cloexec: + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(process.namespace_pidfd, 71) + cloexec.assert_called_once_with(71) + protocol.recvmsg.assert_called_once_with( + 128, + trace.socket.CMSG_SPACE( + array.array("i").itemsize * + trace.LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS), + 1073741824, + ) + + def test_linux_protocol_rejects_truncation_and_unknown_flags(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + for flags in (1073741824 | 32, 1073741824 | 8, 536870912): + with self.subTest(flags=flags): + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"READY", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + flags, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "unexpected flags"): + process._receive_protocol(b"READY") + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_rejects_two_pidfds_in_one_record(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, mock.patch.object( + trace, "_linux_fd_is_close_on_exec") as cloexec, self.assertRaisesRegex( + trace.TraceError, "one namespace pidfd"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + cloexec.assert_not_called() + + def test_linux_protocol_rejects_multiple_rights_records(self) -> None: + first = array.array("i", [71]).tobytes() + second = array.array("i", [72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [ + (trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, first), + (trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, second), + ], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "multiple descriptor records"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_rejects_malformed_rights_payload(self) -> None: + malformed = array.array("i", [71]).tobytes() + b"x" + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, malformed)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "malformed descriptor data"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + close.assert_called_once_with(71) + + def test_linux_protocol_rejects_unexpected_ancillary_and_closes_rights(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [ + (trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights), + (trace.socket.SOL_SOCKET, 12345, b"unexpected"), + ], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "unexpected ancillary data"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + close.assert_called_once_with(71) + + def test_linux_protocol_rejects_pidfd_without_cloexec(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace, "_linux_fd_is_close_on_exec", return_value=False), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "not close-on-exec"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + close.assert_called_once_with(71) + + def test_linux_protocol_fcntl_failure_closes_pidfd(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"PREPARED", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + primary = OSError("fcntl failed") + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace, "_linux_fd_is_close_on_exec", side_effect=primary), mock.patch.object( + trace.os, "close") as close, self.assertRaises( + trace.ExecutionIntegrityError) as raised: + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertIs(raised.exception.primary_error, primary) + self.assertFalse(raised.exception.quiescence_proven) + close.assert_called_once_with(71) + + def test_linux_protocol_wrong_payload_closes_every_received_fd(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"WRONG", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "protocol expected PREPARED"): + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_close_failure_still_closes_remaining_fds(self) -> None: + rights = array.array("i", [71, 72]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"WRONG", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + close_error = OSError("close failed") + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close", side_effect=[close_error, None]) as close, self.assertRaises( + trace.ExecutionIntegrityError) as raised: + process._receive_protocol(b"PREPARED", receive_pidfd=True) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-helper-received-fd-close"], + ) + self.assertFalse(raised.exception.quiescence_proven) + + def test_linux_protocol_rejects_descriptor_on_payload_only_message(self) -> None: + rights = array.array("i", [71]).tobytes() + protocol = mock.Mock() + protocol.recvmsg.return_value = ( + b"READY", + [(trace.socket.SOL_SOCKET, trace.socket.SCM_RIGHTS, rights)], + 1073741824, + None, + ) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object( + trace.socket, "MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + trace.os, "close") as close, self.assertRaisesRegex( + trace.TraceError, "unexpected descriptor"): + process._receive_protocol(b"READY") + close.assert_called_once_with(71) + + def test_native_pidfd_packet_rejects_truncation_unknown_flags_and_oversize(self) -> None: + import socket + + item_size = array.array("i").itemsize + rights = array.array("i", [71]).tobytes() + cases = ( + ((b"ready", [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], 1073741824 | 32, None), 5), + ((b"ready", [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], 536870912, None), 5), + ((b"ready!", [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], 1073741824, None), 5), + ) + for packet, bound in cases: + with self.subTest(flags=packet[2], size=len(packet[0])): + connection = mock.Mock() + connection.recvmsg.return_value = packet + with mock.patch( + "socket.MSG_CMSG_CLOEXEC", 1073741824, create=True), mock.patch.object( + os, "close") as close, self.assertRaises(AssertionError): + receive_native_test_pidfd_packet(connection, bound) + close.assert_called_once_with(71) + connection.recvmsg.assert_called_once_with( + bound + 1, + socket.CMSG_SPACE(item_size), + 1073741824, + ) + + def test_native_pidfd_packet_requires_one_descriptor(self) -> None: + import socket + + rights = array.array("i", [71, 72]).tobytes() + connection = mock.Mock() + connection.recvmsg.return_value = ( + b"ready", + [(socket.SOL_SOCKET, socket.SCM_RIGHTS, rights)], + 0, + None, + ) + with mock.patch.object(os, "close") as close, self.assertRaisesRegex( + AssertionError, "exactly one pidfd"): + receive_native_test_pidfd_packet(connection, len(b"ready")) + self.assertEqual(close.call_args_list, [mock.call(71), mock.call(72)]) + + def test_linux_protocol_startup_failure_reaps_and_closes_streams(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=84, + ) + process.root_pidfd = 90 + primary = trace.TraceError("protocol startup failed") + cleanup = trace._ContainmentCleanup([], True) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_start_linux_native_helper_process", return_value=process), mock.patch.object( + process, "release_exec", side_effect=primary), mock.patch.object( + process, "abort_blocked", return_value=cleanup) as abort, mock.patch.object( + process, "collect_startup_stderr", + return_value=(b"stage=namespace-mount-proc errno=1\n", [])) as collect_stderr, mock.patch.object( + process, "close_streams", return_value=[]) as close_streams, mock.patch.object( + trace.os, "close"), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_linux_native_helper(["approved"], {}) + self.assertIs(raised.exception.primary_error, primary) + self.assertFalse(raised.exception.quiescence_proven) + self.assertIn( + "helper stderr b'stage=namespace-mount-proc errno=1\\n'", + str(raised.exception), + ) + abort.assert_called_once() + collect_stderr.assert_called_once() + close_streams.assert_called_once() + lock.release.assert_called_once() + + def test_linux_startup_stderr_capture_is_bounded_and_closed(self) -> None: + read_fd, write_fd = os.pipe() + os.write(write_fd, b"exact helper diagnostic\n") + os.close(write_fd) + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=read_fd, + ) + stderr, failures = process.collect_startup_stderr() + self.assertEqual(stderr, b"exact helper diagnostic\n") + self.assertEqual(failures, []) + self.assertIsNone(process._stderr_fd) + with self.assertRaises(OSError): + os.fstat(read_fd) + + process._stderr_fd = 84 + oversized = b"x" * (trace.PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES + 1) + with mock.patch.object(trace.os, "read", side_effect=[oversized]), mock.patch.object( + trace.os, "close") as close: + stderr, failures = process.collect_startup_stderr() + self.assertEqual(len(stderr), trace.PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES) + self.assertEqual( + [failure.component for failure in failures], + ["linux-helper-stderr-bounds"], + ) + close.assert_called_once_with(84) + + def test_native_test_report_is_bounded_and_schema_exact(self) -> None: + state = { + "uids": [65534] * 3, + "gids": [65534] * 3, + "groups": [], + "sid": 3, + "pgrp": 3, + "pid": 3, + "caps": ["0000000000000000"] * 5, + "no_new_privs": "1", + "seccomp": "2", + "securebits": 239, + } + encoded = ( + "target:" + json.dumps(state, sort_keys=True, separators=(",", ":")) + ).encode("ascii") + self.assertEqual(decode_native_test_report(encoded), ("target", state)) + self.assertEqual(decode_native_test_report(b"descendant"), ("descendant", None)) + for invalid in ( + b"x" * (NATIVE_TEST_STATE_PACKET_MAX_BYTES + 1), + b"target:{", + b"target:{}", + b"descendant:payload"): + with self.subTest(invalid=invalid[:32]), self.assertRaises(AssertionError): + decode_native_test_report(invalid) + + def test_native_supervisor_stderr_is_preserved_and_closed(self) -> None: + supervisor = subprocess.Popen( + [sys.executable, "-c", "import sys;sys.stderr.write('helper failure')"], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + self.assertEqual(returncode, 0) + self.assertEqual(stderr, "helper failure") + self.assertTrue(supervisor.stderr.closed) + + def test_linux_namespace_authority_precedes_release_completion(self) -> None: + events = [] + lock = mock.Mock() + lock.acquire.return_value = True + process = mock.Mock( + pid=71, + root_pidfd=90, + namespace_pidfd=91, + launch_primary_error=None, + launch_integrity_failures=[], + ) + process.release_exec.side_effect = lambda: events.append("protocol") + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "getgroups", return_value=[]), mock.patch.object( + trace, "_linux_require_pidfd_support"), mock.patch.object( + trace, "_linux_task_ids", return_value={1}), mock.patch.object( + trace, "_start_linux_native_helper_process", + side_effect=lambda *_args: events.append("spawn") or process), mock.patch.object( + trace, "_linux_pidfd_has_exited", return_value=False): + containment = trace._start_linux_native_helper(["approved"], {}) + self.assertEqual(events, ["spawn", "protocol"]) + self.assertEqual(containment.linux_root_pidfd, 90) + self.assertEqual(containment.linux_namespace_pidfd, 91) + self.assertTrue(containment.linux_exec_released) + + def test_linux_native_helper_process_identity_loss_never_signals_numeric_pid(self) -> None: + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + with mock.patch.object(trace.os, "waitpid", side_effect=ChildProcessError), mock.patch.object( + trace.os, "kill") as kill, self.assertRaisesRegex( + trace.TraceError, "identity was lost"): + process.kill() + kill.assert_not_called() + + def test_linux_native_helper_source_binds_supervisor_death_and_namespace_lifecycle(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + self.assertGreaterEqual(helper_source.count("PR_SET_PDEATHSIG"), 2) + self.assertIn("CLONE_NEWPID", helper_source) + self.assertIn("kill(-1, SIGKILL)", helper_source) + self.assertIn("getppid() != expected_parent", helper_source) + self.assertIn("getppid() != 0", helper_source) + self.assertGreaterEqual(helper_source.count("make_isolated_session();"), 2) + self.assertIn("require_parent_death(0);", helper_source) + self.assertIn("require_parent_death(1);", helper_source) + + def test_linux_native_helper_source_reports_bounded_setup_diagnostics(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + diagnostic_source = helper_source[ + helper_source.index("constexpr uint32_t DIAGNOSTIC_MAGIC"): + helper_source.index("bool retained_fd(") + ] + self.assertIn("constexpr size_t DIAGNOSTIC_STAGE_CAPACITY = 48;", diagnostic_source) + self.assertIn("static_assert(sizeof(failure_diagnostic) <= PIPE_BUF);", diagnostic_source) + self.assertIn("record.magic != DIAGNOSTIC_MAGIC", diagnostic_source) + self.assertIn("record.version != DIAGNOSTIC_VERSION", diagnostic_source) + self.assertIn("record.stage_size > DIAGNOSTIC_STAGE_CAPACITY", diagnostic_source) + self.assertIn("invalid setup diagnostic", diagnostic_source) + self.assertIn("count < 8", diagnostic_source) + self.assertIn("pipe2(diagnostic_pipe, O_CLOEXEC | O_NONBLOCK)", helper_source) + self.assertIn( + "run_namespace_init(\n" + " release_pipe[0], ready_pipe[1], mapping_pipe[0], diagnostic_pipe[1],", + helper_source, + ) + self.assertGreaterEqual( + helper_source.count("throw_setup_failure("), + 5, + ) + required_stages = { + "namespace-parent-death", + "namespace-parent-identity", + "namespace-mapping-read", + "namespace-groups-verify", + "namespace-setresgid", + "namespace-setresuid", + "namespace-mount-private", + "namespace-unmount-proc", + "namespace-mount-proc", + "namespace-verify-proc", + "namespace-protect-init", + "namespace-ready", + "target-parent-death", + "target-session", + "target-mapping-read", + "target-groups-verify", + "target-privilege-drop", + "target-isolation-probes", + "target-isolation-ready", + "target-exec", + } + stages = [ + line.split('stage = "', 1)[1].split('"', 1)[0] + for line in helper_source.splitlines() + if 'stage = "' in line + ] + self.assertEqual(len(stages), len(set(stages))) + self.assertTrue(all(0 < len(stage) <= 48 for stage in stages)) + self.assertTrue(all( + stage.startswith(("namespace-", "target-")) + for stage in stages + )) + for stage in required_stages: + self.assertIn(f'"{stage}"', helper_source) + child_source = helper_source[ + helper_source.index("[[noreturn]] void run_target_bootstrap"): + helper_source.index("int run_linux_helper") + ] + self.assertNotIn("_exit(125)", child_source) + self.assertIn("fail_stage(diagnostic_fd, stage, errno);", child_source) + + def test_linux_native_tests_do_not_hide_startup_failures(self) -> None: + source = ( + inspect.getsource(self.test_linux_native_helper_parent_death_boundary) + + inspect.getsource(self.test_linux_native_helper_forbidden_operations_kill_namespace) + ) + self.assertNotIn("skipTest(", source) + self.assertNotIn("'stderr':-3", source) + self.assertGreaterEqual(source.count("finish_native_test_supervisor("), 4) + self.assertGreaterEqual(source.count("self.fail("), 2) + + def test_linux_native_helper_procfs_overmount_is_verified_and_fail_closed(self) -> None: + helper_source = ( + Path(__file__).parents[1] / + "tools/deepseek-v41-trace/linux-containment-helper.cpp" + ).read_text(encoding="ascii") + setup_source = helper_source[ + helper_source.index('stage = "namespace-mount-private"'): + helper_source.index('stage = "namespace-protect-init"') + ] + self.assertIn( + 'umount2("/proc", MNT_DETACH) != 0 && errno != EINVAL', + setup_source, + ) + self.assertIn( + 'mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr)', + setup_source, + ) + self.assertLess( + setup_source.index('stage = "namespace-unmount-proc"'), + setup_source.index('stage = "namespace-mount-proc"'), + ) + self.assertLess( + setup_source.index('stage = "namespace-mount-proc"'), + setup_source.index("verify_private_procfs();"), + ) + verification_source = helper_source[ + helper_source.index("void verify_private_procfs()"): + helper_source.index("void require_initial_signal_state()") + ] + self.assertIn("PROC_SUPER_MAGIC", verification_source) + self.assertIn('readlink("/proc/self"', verification_source) + self.assertIn("size != 1 || self_target[0] != '1'", verification_source) + + @unittest.skipUnless( + sys.platform == "linux" and os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER"), + "native Linux containment helper was not executed on this host", + ) + def test_linux_native_helper_parent_death_boundary(self) -> None: + import array + import socket + + helper = Path(os.environ["DSV41_NATIVE_CONTAINMENT_HELPER"]).resolve(strict=True) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + socket_path = root / "pidfds.sock" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + listener.bind(str(socket_path)) + listener.listen(2) + listener.settimeout(10) + target_source = ( + "import array,ctypes,json,os,socket,sys,time\n" + "libc=ctypes.CDLL(None,use_errno=True)\n" + "def report(label,payload=None):\n" + " s=socket.socket(socket.AF_UNIX,socket.SOCK_SEQPACKET)\n" + " s.connect(sys.argv[1])\n" + " fd=os.pidfd_open(os.getpid())\n" + " rights=array.array('i',[fd])\n" + " data=label if payload is None else label+':'+json.dumps(payload,sort_keys=True)\n" + " s.sendmsg([data.encode('ascii')],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,rights)])\n" + " os.close(fd);s.close()\n" + "status={line.split(':',1)[0]:line.split(':',1)[1].strip()" + " for line in open('/proc/self/status',encoding='ascii') if ':' in line}\n" + "state={'uids':list(os.getresuid()),'gids':list(os.getresgid()),'groups':os.getgroups()," + "'sid':os.getsid(0),'pgrp':os.getpgrp(),'pid':os.getpid()," + "'caps':[status[name] for name in ('CapInh','CapPrm','CapEff','CapBnd','CapAmb')]," + "'no_new_privs':status['NoNewPrivs'],'seccomp':status['Seccomp']," + "'securebits':libc.prctl(27,0,0,0,0)}\n" + "report('target',state)\n" + "if os.fork()==0:\n" + " time.sleep(.2);report('descendant')\n" + " while True: time.sleep(1)\n" + "while True: time.sleep(1)\n" + ) + supervisor_source = ( + "import os,sys,time\n" + "from pathlib import Path\n" + "import trace_format as trace\n" + "helper=Path(sys.argv[1])\n" + "target=os.open(sys.executable,os.O_RDONLY)\n" + "helper_fd=os.open(helper,os.O_RDONLY)\n" + "launch={'executable':f'/proc/self/fd/{target}'," + "'pass_fds':(target,)," + "'_containment_helper_path':f'/proc/self/fd/{helper_fd}'," + "'_containment_helper_descriptor':helper_fd," + "'stdout':-3}\n" + "containment=trace._start_linux_native_helper(" + "[sys.executable,'-c',sys.argv[2],sys.argv[3],str(os.getpgrp())],launch)\n" + "while True: time.sleep(1)\n" + ) + environment = { + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(Path(__file__).parents[1] / "tools/deepseek-v41-trace"), + } + supervisor = subprocess.Popen( + [sys.executable, "-c", supervisor_source, str(helper), target_source, str(socket_path)], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + pidfds = [] + supervisor_reaped = False + try: + labels = set() + target_state = None + while len(pidfds) < 2: + try: + connection, _address = listener.accept() + except TimeoutError: + if supervisor.poll() is not None: + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + supervisor_reaped = True + self.fail( + f"native helper failed before target report " + f"(exit {returncode}); stderr={stderr!r}") + raise + with connection: + data, pidfd = receive_native_test_pidfd_packet( + connection, NATIVE_TEST_STATE_PACKET_MAX_BYTES) + pidfds.append(pidfd) + label, state = decode_native_test_report(data) + labels.add(label) + if state is not None: + target_state = state + self.assertEqual(labels, {"target", "descendant"}) + self.assertIsNotNone(target_state) + self.assertEqual(target_state["uids"], [65534] * 3) + self.assertEqual(target_state["gids"], [65534] * 3) + self.assertEqual(target_state["groups"], []) + self.assertEqual(target_state["sid"], target_state["pid"]) + self.assertEqual(target_state["pgrp"], target_state["pid"]) + self.assertEqual(target_state["caps"], ["0000000000000000"] * 5) + self.assertEqual(target_state["no_new_privs"], "1") + self.assertEqual(target_state["seccomp"], "2") + self.assertEqual(target_state["securebits"], 239) + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=True) + supervisor_reaped = True + self.assertEqual(returncode, -trace.signal.SIGKILL, stderr) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and not all( + trace._linux_pidfd_has_exited(pidfd) for pidfd in pidfds): + time.sleep(0.02) + self.assertTrue(all( + trace._linux_pidfd_has_exited(pidfd) for pidfd in pidfds)) + finally: + for pidfd in pidfds: + if not trace._linux_pidfd_has_exited(pidfd): + trace._linux_signal_pidfd(pidfd, trace.signal.SIGKILL) + os.close(pidfd) + if not supervisor_reaped: + finish_native_test_supervisor(supervisor, kill_if_running=True) + listener.close() + + @unittest.skipUnless( + sys.platform == "linux" and os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER"), + "native Linux containment helper was not executed on this host", + ) + def test_linux_native_helper_forbidden_operations_kill_namespace(self) -> None: + import array + import socket + + helper = Path(os.environ["DSV41_NATIVE_CONTAINMENT_HELPER"]).resolve(strict=True) + target_source = ( + "import array,ctypes,os,socket,sys,time\n" + "libc=ctypes.CDLL(None,use_errno=True)\n" + "s=socket.socket(socket.AF_UNIX,socket.SOCK_SEQPACKET)\n" + "s.connect(sys.argv[1])\n" + "fd=os.pidfd_open(os.getpid())\n" + "rights=array.array('i',[fd])\n" + "s.sendmsg([b'ready'],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,rights)])\n" + "os.close(fd);s.close();time.sleep(.1)\n" + "attack=sys.argv[2]\n" + "if attack=='ptrace': libc.ptrace(16,1,0,0)\n" + "elif attack=='ptrace_poke': libc.ptrace(5,1,0,0)\n" + "elif attack=='process_vm_readv': libc.process_vm_readv(1,0,0,0,0,0)\n" + "elif attack=='process_vm_writev': libc.process_vm_writev(1,0,0,0,0,0)\n" + "elif attack=='pdeathsig': libc.prctl(1,0,0,0,0)\n" + "elif attack=='pid1_stop': libc.kill(1,19)\n" + "elif attack=='outer_group': libc.kill(-int(sys.argv[3]),0)\n" + "elif attack=='setuid': libc.setresuid(0,0,0)\n" + "elif attack=='setpgid': libc.setpgid(0,0)\n" + "elif attack=='setsid': libc.setsid()\n" + "else: raise SystemExit(91)\n" + "open(sys.argv[4],'w',encoding='ascii').write('survived')\n" + "while True: time.sleep(1)\n" + ) + supervisor_source = ( + "import os,sys\n" + "from pathlib import Path\n" + "import trace_format as trace\n" + "helper=Path(sys.argv[1])\n" + "target=os.open(sys.executable,os.O_RDONLY)\n" + "helper_fd=os.open(helper,os.O_RDONLY)\n" + "launch={'executable':f'/proc/self/fd/{target}'," + "'pass_fds':(target,)," + "'_containment_helper_path':f'/proc/self/fd/{helper_fd}'," + "'_containment_helper_descriptor':helper_fd," + "'stdout':-3}\n" + "containment=trace._start_linux_native_helper(" + "[sys.executable,'-c',sys.argv[2],sys.argv[3],sys.argv[4]," + "str(os.getpgrp()),sys.argv[5]],launch)\n" + "containment.process.communicate(timeout=10)\n" + "raise SystemExit(containment.process.returncode)\n" + ) + environment = { + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(Path(__file__).parents[1] / "tools/deepseek-v41-trace"), + } + attacks = ( + "ptrace", + "ptrace_poke", + "process_vm_readv", + "process_vm_writev", + "pdeathsig", + "pid1_stop", + "outer_group", + "setuid", + "setpgid", + "setsid", + ) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for attack in attacks: + with self.subTest(attack=attack): + socket_path = root / f"{attack}.sock" + marker = root / f"{attack}.survived" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + listener.bind(str(socket_path)) + listener.listen(1) + listener.settimeout(10) + supervisor = subprocess.Popen( + [ + sys.executable, + "-c", + supervisor_source, + str(helper), + target_source, + str(socket_path), + attack, + str(marker), + ], + env=environment, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + target_pidfd = None + supervisor_reaped = False + try: + try: + connection, _address = listener.accept() + except TimeoutError: + if supervisor.poll() is not None: + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + supervisor_reaped = True + self.fail( + f"native helper failed before attack report " + f"(exit {returncode}); stderr={stderr!r}") + raise + with connection: + data, target_pidfd = receive_native_test_pidfd_packet( + connection, len(b"ready")) + self.assertEqual(data, b"ready") + returncode, stderr = finish_native_test_supervisor( + supervisor, kill_if_running=False) + supervisor_reaped = True + self.assertEqual(returncode, 125, stderr) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and not trace._linux_pidfd_has_exited( + target_pidfd): + time.sleep(0.02) + self.assertTrue(trace._linux_pidfd_has_exited(target_pidfd)) + self.assertFalse(marker.exists()) + finally: + if target_pidfd is not None: + if not trace._linux_pidfd_has_exited(target_pidfd): + trace._linux_signal_pidfd(target_pidfd, trace.signal.SIGKILL) + os.close(target_pidfd) + if not supervisor_reaped: + finish_native_test_supervisor(supervisor, kill_if_running=True) + listener.close() + + def test_linux_native_helper_stream_close_reports_every_failure(self) -> None: + protocol = mock.Mock() + protocol.close.side_effect = OSError("protocol") + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=protocol, + stdin_fd=82, + stdout_fd=83, + stderr_fd=84, + ) + with mock.patch.object(trace.os, "close", side_effect=[ + OSError("stdin"), None, OSError("stderr")]): + failures = process.close_streams() + self.assertEqual( + [failure.component for failure in failures], + [ + "linux-process-fd-close:protocol", + "linux-process-fd-close:stdin", + "linux-process-fd-close:stderr", + ], + ) + + def test_linux_native_helper_rejects_controls_before_spawn(self) -> None: + for launch in ({"shell": True}, {"close_fds": False}, {"cwd": "/tmp"}): + with self.subTest(controls=sorted(launch)), mock.patch.object( + trace.os, "posix_spawn") as spawn, self.assertRaises(trace.TraceError): + trace._start_linux_native_helper_process( + ["/bin/true"], + { + "_containment_helper_path": "/approved/helper", + "_containment_helper_descriptor": 40, + **launch, + }, + ) + spawn.assert_not_called() + + def test_unproven_posix_containment_fails_closed_before_setsid_escape(self) -> None: + if sys.platform == "linux": + self.skipTest("Linux uses subreaper and pidfd containment") + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + marker = root / "escaped" + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + f"printf started > '{marker}'\n" + "sleep 30\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + with isolated_test_install_trust(process_containment=False): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + with self.assertRaisesRegex( + run_ds4.TraceError, "proven process containment is unavailable"): + run_ds4.run_exporter_command( + [str(exporter), str(marker)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=1, + check=False, + capture_output=True, + ) + self.assertFalse(marker.exists()) + + def test_windows_job_containment_is_suspended_before_assignment_and_resume(self) -> None: + start_source = inspect.getsource(trace._start_windows_job_process) + create_source = inspect.getsource(trace._create_windows_kill_job) + cleanup_source = inspect.getsource(trace._terminate_process_tree) + self.assertLess(start_source.index("subprocess.Popen"), start_source.index("AssignProcessToJobObject")) + self.assertLess(start_source.index("AssignProcessToJobObject"), start_source.index("ResumeThread")) + self.assertIn("WINDOWS_CREATE_SUSPENDED", start_source) + self.assertIn("WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE", create_source) + self.assertIn("SetInformationJobObject", create_source) + self.assertIn("TerminateJobObject", cleanup_source) + self.assertIn("QueryInformationJobObject", inspect.getsource(trace._windows_job_active_processes)) + self.assertIn("CloseHandle", inspect.getsource(trace._close_process_containment)) + sentinel = mock.Mock() + launch = {} + with mock.patch.object(trace.sys, "platform", "win32"), mock.patch.object( + trace, "_start_windows_job_process", return_value=sentinel) as start: + self.assertIs(trace._start_contained_process(["approved"], launch), sentinel) + start.assert_called_once_with(["approved"], launch) + + def test_windows_job_assignment_happens_before_resume(self) -> None: + events = [] + process = mock.Mock(pid=91) + process._handle = 92 + kernel32 = mock.Mock() + kernel32.AssignProcessToJobObject.side_effect = lambda *_args: events.append("assign") or 1 + kernel32.ResumeThread.side_effect = lambda *_args: events.append("resume") or 0 + kernel32.CloseHandle.side_effect = lambda *_args: events.append("close") or 1 + with mock.patch.object( + trace.subprocess, + "Popen", + side_effect=lambda *_args, **_kwargs: events.append("popen") or process, + ) as popen, mock.patch.object( + trace, + "_create_windows_kill_job", + side_effect=lambda: events.append("job") or 93, + ), mock.patch.object( + trace, "_windows_kernel32", return_value=kernel32), mock.patch.object( + trace, + "_open_windows_process_thread", + side_effect=lambda _pid: events.append("thread") or 94, + ): + containment = trace._start_windows_job_process(["approved"], {}) + self.assertEqual(events, ["popen", "job", "assign", "thread", "resume", "close"]) + self.assertEqual( + popen.call_args.kwargs["creationflags"] & trace.WINDOWS_CREATE_SUSPENDED, + trace.WINDOWS_CREATE_SUSPENDED, + ) + self.assertEqual(containment.job_handle, 93) + self.assertTrue(containment.windows_job_assigned) + self.assertTrue(containment.windows_process_resumed) + + def test_windows_job_assignment_failure_kills_and_reaps_exact_child(self) -> None: + class OwnedHandle: + def __init__(self, value: int): + self.value = value + self.close_count = 0 + + def __int__(self) -> int: + return self.value + + def Close(self) -> None: + if self.close_count: + raise AssertionError("process handle closed twice") + self.close_count += 1 + + process = mock.Mock(pid=91) + process._handle = OwnedHandle(92) + owned_handle = process._handle + kernel32 = mock.Mock() + kernel32.AssignProcessToJobObject.return_value = 0 + kernel32.CloseHandle.return_value = 1 + with mock.patch.object( + trace.subprocess, "Popen", return_value=process), mock.patch.object( + trace, "_create_windows_kill_job", return_value=93), mock.patch.object( + trace, "_windows_kernel32", return_value=kernel32), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace._start_windows_job_process(["approved"], {}) + self.assertFalse(raised.exception.quiescence_proven) + process.kill.assert_called_once() + process.wait.assert_called_once_with(timeout=trace.PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + kernel32.TerminateJobObject.assert_not_called() + self.assertEqual(kernel32.CloseHandle.call_args_list, [mock.call(93)]) + self.assertEqual(owned_handle.close_count, 1) + self.assertIsNone(process._handle) + + def test_windows_process_handle_ownership_closes_once(self) -> None: + class OwnedHandle: + def __init__(self): + self.close_count = 0 + + def Close(self) -> None: + if self.close_count: + raise AssertionError("recycled process handle closed") + self.close_count += 1 + + process = mock.Mock() + owned_handle = OwnedHandle() + process._handle = owned_handle + with mock.patch.object(trace, "_windows_kernel32") as kernel32: + trace._close_windows_process_handle(process) + trace._close_windows_process_handle(process) + self.assertEqual(owned_handle.close_count, 1) + self.assertIsNone(process._handle) + kernel32.assert_not_called() + + def test_linux_native_helper_closes_after_timeout_and_target_exception(self) -> None: + for primary in ( + subprocess.TimeoutExpired(["approved"], 1), + OSError("target failed"), + ): + with self.subTest(primary=type(primary).__name__): + lock = mock.Mock() + process = mock.Mock(pid=71) + process.communicate.side_effect = primary + containment = trace._ProcessContainment( + process=process, + linux_root_pidfd=90, + linux_namespace_pidfd=91, + linux_lock_held=True, + linux_exec_released=True, + ) + cleanup = trace._ContainmentCleanup([], True) + with mock.patch.object( + trace, "_start_contained_process", return_value=containment), mock.patch.object( + trace, "_terminate_process_tree", return_value=cleanup), mock.patch.object( + trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace.os, "close"), mock.patch.object( + trace, "_linux_pidfd_has_exited", return_value=True): + result = trace._run_contained_process( + ["approved"], + label="approved executable", + timeout=1, + input_data=None, + launch={}, + ) + close_failures = trace._close_process_containment( + containment, + quiescence_proven=result.quiescence_proven, + ) + self.assertIs(result.primary_error, primary) + self.assertEqual(close_failures, []) + lock.release.assert_called_once() + + def test_linux_native_helper_teardown_requires_tree_quiescence(self) -> None: + lock = mock.Mock() + containment = trace._ProcessContainment( + process=mock.Mock(), + linux_lock_held=True, + ) + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", False), mock.patch.object( + trace, "_LINUX_POISONED_CONTAINMENT", None): + failures = trace._close_process_containment( + containment, + quiescence_proven=False, + ) + self.assertTrue(trace._LINUX_HELPER_POISONED) + self.assertIs(trace._LINUX_POISONED_CONTAINMENT, containment) + self.assertEqual([failure.component for failure in failures], ["linux-helper-teardown"]) + lock.release.assert_not_called() + + def test_linux_native_helper_rejects_concurrent_reuse(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = False + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( + trace.TraceError, "already active"): + trace._start_linux_native_helper(["approved"], {}) + start.assert_not_called() + lock.release.assert_not_called() + + def test_linux_poisoned_helper_supervisor_rejects_reuse(self) -> None: + lock = mock.Mock() + lock.acquire.return_value = True + with mock.patch.object(trace, "_LINUX_HELPER_LOCK", lock), mock.patch.object( + trace, "_LINUX_HELPER_POISONED", True), mock.patch.object( + trace, "_start_linux_native_helper_process") as start, self.assertRaisesRegex( + trace.TraceError, "not reusable"): + trace._start_linux_native_helper(["approved"], {}) + start.assert_not_called() + lock.release.assert_called_once() + + def test_linux_missing_helper_completion_blocks_containment_gate(self) -> None: + process = trace._LinuxNativeHelperProcess( + ["approved"], + 71, + protocol_socket=mock.Mock(), + stdin_fd=None, + stdout_fd=None, + stderr_fd=None, + ) + process.returncode = 0 + containment = trace._ProcessContainment( + process=process, + linux_root_pidfd=90, + linux_namespace_pidfd=91, + linux_lock_held=True, + linux_exec_released=True, + ) + with mock.patch.object( + process, "communicate", return_value=(b"", b"")), mock.patch.object( + trace, "_linux_signal_owned_children", return_value=[]), mock.patch.object( + trace, "_linux_pidfd_has_exited", return_value=True): + cleanup = trace._terminate_process_tree(containment) + self.assertFalse(cleanup.quiescence_proven) + self.assertEqual( + [failure.component for failure in cleanup.failures], + ["linux-helper-completion"], + ) + + def test_containment_helper_policy_mutations_fail_closed(self) -> None: + for field, value in ( + ("revision", "b" * 40), + ("filename", "other-helper"), + ("sha256", "not-a-digest"), + ("version", 1), + ("launcher_policy", "unbound"), + ("supplementary_groups", [44])): + with self.subTest(field=field): + policy = fixture_prompt_builder_policy(b"prompt") + policy["containment_helper"][field] = value + with self.assertRaisesRegex(trace.TraceError, "containment helper receipt"): + trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + + def test_containment_handle_close_failure_forces_quiescence_false(self) -> None: + with tempfile.TemporaryDirectory() as temp: + install = Path(temp).resolve() / "install" + executable = install / "bin" / "approved" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"approved") + executable.chmod(0o555) + policy = { + "install_root": str(install), + "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0, + "runtime_receipt": {"components": []}, + } + result = subprocess.CompletedProcess([str(executable)], 0, b"", b"") + contained = trace._ContainedRun(result, None, [], mock.Mock(), True, True) + close_failure = trace._IntegrityFailure( + "linux-root-pidfd-close", OSError("close failed")) + with isolated_test_install_trust(), mock.patch.object( + trace, "_run_contained_process", return_value=contained), mock.patch.object( + trace, "_close_process_containment", return_value=[close_failure]), self.assertRaises( + trace.ExecutionIntegrityError) as raised: + trace.run_approved_executable( + [str(executable)], + path=executable, + runtime_policy=policy, + expected_path=str(executable), + expected_sha256=trace.sha256_file(executable), + label="approved executable", + check=False, + capture_output=True, + ) + self.assertFalse(raised.exception.quiescence_proven) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["linux-root-pidfd-close"], + ) + + def test_ds4_writable_root_blocks_restore_before_postcheck(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + Path(policy["install_root"]).chmod(0o777) + try: + with mock.patch.object(trace, "_run_contained_process") as execute, self.assertRaisesRegex( + run_ds4.TraceError, "path is mutable|distinct"): + run_ds4.run_exporter_command( + [str(exporter)], + exporter=exporter, + exporter_identity=identity, + exporter_policy=policy, + timeout_seconds=30, + ) + execute.assert_not_called() + finally: + Path(policy["install_root"]).chmod(0o755) + + def test_ds4_exporter_rejects_hardlinks_and_dependency_substitution(self) -> None: + for mutation, message in ( + ("hard-linked-exporter", "one-link"), + ("hard-linked-dependency", "one-link"), + ("dependency-substitution", "SHA-256 differs"), + ("symlinked-exporter", "differs|canonical|aliases"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + library = Path(policy["install_root"]) / "lib" / policy["runtime_receipt"]["components"][0]["filename"] + if mutation == "hard-linked-exporter": + os.link(exporter, root / "exporter-alias") + elif mutation == "hard-linked-dependency": + os.link(library, root / "library-alias") + elif mutation == "dependency-substitution": + library.chmod(0o755) + library.write_bytes(b"substituted") + library.chmod(0o555) + else: + alias = root / "exporter-alias" + alias.symlink_to(exporter) + exporter = alias + with isolated_test_install_trust(), mock.patch.object( + trace, "_run_contained_process") as execute, self.assertRaisesRegex( + trace.TraceError, message): + trace.run_approved_executable( + [str(exporter)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + execute.assert_not_called() + + def test_ds4_exporter_rejects_command_path_substitution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + replacement = Path(temp) / "replacement" + replacement.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") + replacement.chmod(0o555) + with isolated_test_install_trust(), mock.patch.object( + trace, "_run_contained_process") as execute, self.assertRaisesRegex( + trace.TraceError, "command path differs"): + trace.run_approved_executable( + [str(replacement)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + execute.assert_not_called() + + def test_ds4_exporter_rejects_execution_owned_install(self) -> None: + with tempfile.TemporaryDirectory() as temp: + policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve()) + with mock.patch.object(trace, "_run_contained_process") as execute, self.assertRaisesRegex( + trace.TraceError, "owner must be distinct"): + trace.run_approved_executable( + [str(exporter)], + path=exporter, + runtime_policy=policy, + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + execute.assert_not_called() + + def test_ds4_policy_and_receipt_mutation_change_approval_identity(self) -> None: + policy = copy.deepcopy(DS4_EXPORTER_POLICY) + _approved, original_sha256 = trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + ) + policy["runtime_receipt"]["components"][0]["sha256"] = "c" * 64 + _mutated, mutated_sha256 = trace.ds4_exporter_approval( + TEST_DS4_EXPORTER_POLICY_ID, + policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + ) + self.assertNotEqual(original_sha256, mutated_sha256) + record = manifest("ds4") + with self.assertRaisesRegex(trace.TraceError, "ds4 exporter approval"): + trace.validate_execution_authorization( + record, + policy=self.test_signers[self.signer_principals["ds4"]], + expected_lane=trace.ORACLE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["ds4"], + candidate_exporter_policies={}, + ds4_exporter_policies={TEST_DS4_EXPORTER_POLICY_ID: policy}, + prompt_builder_policies={ + TEST_PROMPT_BUILDER_POLICY_ID: fixture_prompt_builder_policy(b"abc"), + }, + expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=TEST_DS4_EXPORTER_POLICY_ID, + expected_prompt_builder_policy_id=TEST_PROMPT_BUILDER_POLICY_ID, + expected_approval_policy_sha256="e" * 64, + expected_verifier_revision="a" * 40, + verification_unix=TEST_AUTH_ISSUED, + seen_run_ids=None, + ) + + def test_install_trust_evidence_rejects_mutability_claims(self) -> None: + policy = fixture_prompt_builder_policy(b"prompt") + mutations = { + "same-owner": lambda value: value.update({"execution_uid": value["owner_uid"]}), + "writable-directory": lambda value: value["directories"][-1].update({"mode": 0o777}), + "directory-acl": lambda value: value["directories"][-1].update({"acl_entries": True}), + "writable-file": lambda value: value["files"][0].update({"mode": 0o755}), + "hard-linked-file": lambda value: value["files"][0].update({"link_count": 2}), + "file-acl": lambda value: value["files"][0].update({"acl_entries": True}), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + trust = fixture_install_trust(policy) + mutate(trust) + with self.assertRaisesRegex(trace.TraceError, "install trust"): + trace.validate_install_trust_evidence(trust, policy) + + def test_prompt_builder_rejects_runtime_receipt_before_execution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + policy = fixture_prompt_builder_policy( + b"prompt", + builder_path=str(builder), + builder_sha256=trace.sha256_file(builder), + source_root=str(Path(__file__).parents[1].resolve()), + ) + materialize_policy_runtime(policy) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + runtime_component = policy["runtime_receipt"]["components"][0] + runtime_path = Path(policy["install_root"]) / "lib" / runtime_component["filename"] + runtime_path.chmod(0o644) + runtime_path.write_bytes(b"changed") + with isolated_test_install_trust(), mock.patch.object( + run_matrix, "run_approved_executable") as execute, self.assertRaisesRegex( + run_matrix.TraceError, "runtime component .* (immutable|SHA-256)"): + run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=policy, + builder_policy_sha256=policy_sha256, + model=root / "model.gguf", + corpus=root / "corpus.txt", + source_corpus=Path(__file__).parents[1] / "tests" / "corpus" / "correctness-prose.txt", + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=root / "prompt.txt", + context=3, + decode_steps=1, + ) + execute.assert_not_called() + + def test_prompt_builder_rejects_output_binary_and_corpus_mutation(self) -> None: + for mutation, message in ( + ("output", "output differs from external approval"), + ("builder", "immutable|SHA-256 differs from external approval"), + ("corpus", "corpus changed during execution"), + ("runtime", "runtime component .*immutable|runtime component SHA-256 differs"), + ("runtime-load", "loaded runtime libraries differ from external approval"), + ("tokenizer", "tokenizer policy"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" + model = root / "model.gguf" + corpus = root / "corpus.txt" + source_root = Path(__file__).parents[1].resolve() + source_corpus = source_root / "tests" / "corpus" / "correctness-prose.txt" + output = root / "prompt.txt" + tmpdir = root / "tmp" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + model.write_bytes(b"model") + shutil.copyfile(source_corpus, corpus) + tmpdir.mkdir() + approved_output = b"approved" if mutation == "output" else b"prompt" + policy = fixture_prompt_builder_policy( + approved_output, + builder_path=str(builder), + builder_sha256=trace.sha256_file(builder), + source_root=str(source_root), + ) + materialize_policy_runtime(policy) + _validated, policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: policy}, + ) + with isolated_test_install_trust(): + initial_identity = run_matrix.approved_executable_identity( + builder, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="prompt builder", + ) + + def run_builder(command, **_kwargs): + if "--dsv41-attest-build" in command: + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps(fixture_runtime_build(policy)), + "", + ), + initial_identity, + ) + output.write_bytes(b"prompt") + if mutation == "builder": + builder.chmod(0o755) + builder.write_bytes(b"changed") + if mutation == "corpus": + corpus.write_bytes(b"changed") + if mutation == "runtime": + runtime_component = policy["runtime_receipt"]["components"][0] + runtime_path = Path(policy["install_root"]) / "lib" / runtime_component["filename"] + runtime_path.chmod(0o644) + runtime_path.write_bytes(b"changed") + runtime_build = fixture_runtime_build(policy) + tokenizer = copy.deepcopy(policy["tokenizer"]) + if mutation == "runtime-load": + runtime_build["runtime_libraries"][0]["path"] = "/tmp/unapproved.so" + runtime_build["runtime_libraries_post"][0]["path"] = "/tmp/unapproved.so" + if mutation == "tokenizer": + tokenizer["parse_special"] = False + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps({ + "target_tokens": 2, + "actual_tokens": 2, + "byte_count": 6, + "tokenizer": tokenizer, + "runtime_build": runtime_build, + "temporary_directory": str(tmpdir), + }), + "", + ), + initial_identity, + ) + + with isolated_test_install_trust(), mock.patch.dict( + os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( + run_matrix, "run_approved_executable", side_effect=run_builder), self.assertRaisesRegex( + (RuntimeError, run_matrix.TraceError), message): + run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=policy, + builder_policy_sha256=policy_sha256, + model=model, + corpus=corpus, + source_corpus=source_corpus, + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=output, + context=3, + decode_steps=1, + ) + + def test_signed_approval_binding_tamper_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + trace_manifest = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + trace_manifest["authorization"]["approvals"]["candidate_exporter"]["sha256"] = "f" * 64 + (root / trace.MANIFEST_NAME).write_text( + trace.canonical_json(trace_manifest) + "\n", encoding="ascii") + with self.assertRaisesRegex( + trace.TraceError, "candidate exporter approval differs from external policy"): + self._seal_test_bundle(root) + + def test_tokenizer_policy_contradictions_are_rejected_before_signing(self) -> None: + mutations = { + "add-bos-and-parse-special": { + "add_bos": False, + "parse_special": False, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": False, + "require_round_trip": True, + }, + "add-bos": { + "add_bos": False, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": False, + "require_round_trip": True, + }, + "detokenize-special": { + "add_bos": True, + "parse_special": True, + "detokenize_special": False, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": True, + }, + "bos-removal": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": False, + "require_round_trip": True, + }, + "round-trip": { + "add_bos": True, + "parse_special": True, + "detokenize_special": True, + "remove_leading_bos_before_detokenize": True, + "require_round_trip": False, + }, + } + for name, tokenizer in mutations.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + record = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + record["config"]["tokenizer"] = tokenizer + (root / trace.MANIFEST_NAME).write_text( + trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "tokenizer"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + valid = fixture_prompt_builder_policy(b"prompt")["tokenizer"] + malformed = { + "missing": {key: value for key, value in valid.items() if key != "parse_special"}, + "extra": {**valid, "unknown": False}, + "non-boolean": {**valid, "add_bos": 1}, + } + for name, tokenizer in malformed.items(): + with self.subTest(name=name), self.assertRaisesRegex(trace.TraceError, "tokenizer policy"): + trace.validate_tokenizer_policy(tokenizer) + + def test_prompt_provenance_tokenizer_mutation_is_rejected_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + old_path = root / record["prompt"]["provenance"]["path"] + provenance = trace.strict_json_loads(old_path.read_text(encoding="ascii")) + provenance["tokenizer"]["add_bos"] = False + provenance["tokenizer"]["remove_leading_bos_before_detokenize"] = False + data = (trace.canonical_json(provenance) + "\n").encode("ascii") + digest = trace.sha256_bytes(data) + new_path = root / "provenance" / f"{digest}.json" + new_path.write_bytes(data) + old_path.unlink() + record["prompt"]["provenance"] = { + "path": f"provenance/{digest}.json", + "sha256": digest, + } + manifest_path.write_text(trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "prompt provenance tokenizer"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + def test_prompt_provenance_runtime_closure_mutation_is_rejected_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + old_path = root / record["prompt"]["provenance"]["path"] + provenance = trace.strict_json_loads(old_path.read_text(encoding="ascii")) + provenance["builder_runtime_build"]["runtime_libraries"][0]["path"] = "/tmp/unapproved.so" + provenance["builder_runtime_build"]["runtime_libraries_post"][0]["path"] = "/tmp/unapproved.so" + provenance["builder_runtime_build_sha256"] = trace.sha256_bytes( + trace.canonical_json(provenance["builder_runtime_build"]).encode("ascii")) + data = (trace.canonical_json(provenance) + "\n").encode("ascii") + digest = trace.sha256_bytes(data) + new_path = root / "provenance" / f"{digest}.json" + new_path.write_bytes(data) + old_path.unlink() + record["prompt"]["provenance"] = { + "path": f"provenance/{digest}.json", + "sha256": digest, + } + manifest_path.write_text(trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "loaded runtime libraries"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + def test_authorization_tokenizer_hash_mutation_is_rejected_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + record = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + record["authorization"]["tokenizer_policy_sha256"] = "f" * 64 + manifest_path.write_text(trace.canonical_json(record) + "\n", encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "tokenizer policy differs"): + self._seal_test_bundle(root) + self.assertFalse((root / trace.SIGNATURE_NAME).exists()) + + def test_seal_rejects_protected_bundle_mutations(self) -> None: + with tempfile.TemporaryDirectory() as temp: + baseline = Path(temp) / "baseline" + with trace.TraceBundleWriter(baseline, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(baseline) + self._read_sealed_bundle(baseline) + + mutations = { + "manifest": lambda root: (root / trace.MANIFEST_NAME).write_bytes( + (root / trace.MANIFEST_NAME).read_bytes().replace(b'"event_count":259', b'"event_count":258')), + "events": lambda root: (root / trace.EVENTS_NAME).write_bytes( + (root / trace.EVENTS_NAME).read_bytes().replace(b'"step":0', b'"step":1', 1)), + "audit": lambda root: next((root / "audits").rglob("*.json")).write_bytes(b"{}\n"), + "blob": lambda root: next((root / trace.BLOBS_DIR).iterdir()).write_bytes(b"changed"), + "provenance": lambda root: next((root / "provenance").iterdir()).write_bytes(b"{}\n"), + "added": lambda root: (root / "unexpected").write_bytes(b"x"), + "signature": lambda root: (root / trace.SIGNATURE_NAME).write_bytes( + (root / trace.SIGNATURE_NAME).read_bytes().replace(b"SSH SIGNATURE", b"SSH SIGNATURX", 1)), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + root = Path(temp) / name + shutil.copytree(baseline, root) + mutate(root) + with self.assertRaises(trace.TraceError): + self._read_sealed_bundle(root) + + def test_seal_binds_runtime_lane_challenge_and_run_id(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(root) + + with self.assertRaisesRegex(trace.TraceError, "external challenge"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime("llama.cpp", expected_challenge="e" * 64), + ) + with self.assertRaisesRegex(trace.TraceError, "external run ID"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime( + "llama.cpp", expected_run_id="strix-llama-other-run"), + ) + with self.assertRaisesRegex(trace.TraceError, "expected execution lane"): + wrong_lane = trace.TraceVerifier.for_tests( + self.signer_principal, + self.test_signers[self.signer_principal]["public_key"], + lane=trace.ORACLE_LANE, + runtime="ds4", + runtime_profile="apple-metal", + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["ds4"], + verification_unix=int(time.time()), + ssh_keygen=self.ssh_keygen, + ) + self._trace_bundle_class(root, verifier=wrong_lane) + + seen_run_ids: set[str] = set() + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime("llama.cpp", seen_run_ids=seen_run_ids), + ) + with self.assertRaisesRegex(trace.TraceError, "reused"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime("llama.cpp", seen_run_ids=seen_run_ids), + ) + + with self.assertRaisesRegex(trace.TraceError, "expired"): + self._trace_bundle_class( + root, + verifier=self._verifier_for_runtime( + "llama.cpp", verification_unix=TEST_AUTH_EXPIRES + 1), + ) + + def test_seal_rejects_nonportable_signed_paths(self) -> None: + invalid_paths = ( + "../escape.json", + "./provenance.json", + "audit//record.json", + r"audit\record.json", + "C:/audit/record.json", + "//server/share.json", + "%2e%2e/escape.json", + "audit/\x01.json", + "audit/\N{LATIN SMALL LETTER E WITH ACUTE}.json", + ) + for index, invalid in enumerate(invalid_paths): + with self.subTest(path=invalid), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + record = manifest() + record["prompt"]["provenance"]["path"] = invalid + with trace.TraceBundleWriter(root, record) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "trace path"): + self._seal_test_bundle(root) + + def test_signing_rejects_bundle_key_and_public_key_fallback(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + bundled_key = root / "private-key" + shutil.copyfile(self.signing_key, bundled_key) + bundled_key.chmod(0o600) + with self.assertRaisesRegex(trace.TraceError, "outside the bundle"): + trace.seal_bundle( + root, + private_key=bundled_key, + principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + public_key_path = self.signing_key.with_suffix(".pub") + public_key_path.chmod(0o600) + with self.assertRaisesRegex(trace.TraceError, "derive|match"): + trace.validate_signing_identity( + public_key_path, + self.signer_principal, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + with mock.patch.dict(os.environ, {"SSH_AUTH_SOCK": "/tmp/attacker-agent"}): + trace.validate_signing_identity( + self.signing_key, + self.signer_principal, + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + def test_signer_policy_rejects_open_ssh_options_certificates_and_extra_fields(self) -> None: + policy = copy.deepcopy(self.test_signers[self.signer_principal]) + public_key = policy["public_key"] + invalid_keys = ( + f"cert-authority {public_key}", + public_key.replace("ssh-ed25519", "ssh-ed25519-cert-v01@openssh.com", 1), + f"{public_key} comment", + f"{public_key}\n{public_key}", + ) + for public_key_value in invalid_keys: + with self.subTest(public_key=public_key_value): + invalid_policy = copy.deepcopy(policy) + invalid_policy["public_key"] = public_key_value + with self.assertRaisesRegex(trace.TraceError, "OpenSSH Ed25519 key"): + trace.validate_signing_identity( + self.signing_key, + self.signer_principal, + trusted_signers={self.signer_principal: invalid_policy}, + ssh_keygen=self.ssh_keygen, + ) + with self.assertRaisesRegex(trace.TraceError, "principal is invalid"): + trace.validate_signing_identity( + self.signing_key, + "*", + trusted_signers={"*": policy}, + ssh_keygen=self.ssh_keygen, + ) + + def test_seal_rejects_noncanonical_duplicate_and_incomplete_files(self) -> None: + with tempfile.TemporaryDirectory() as temp: + baseline = Path(temp) / "baseline" + with trace.TraceBundleWriter(baseline, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(baseline) + + noncanonical = Path(temp) / "noncanonical" + shutil.copytree(baseline, noncanonical) + record = json.loads((noncanonical / trace.MANIFEST_NAME).read_text(encoding="ascii")) + (noncanonical / trace.MANIFEST_NAME).write_text( + json.dumps(record, sort_keys=True, indent=2) + "\n", + encoding="ascii", + ) + with self.assertRaisesRegex(trace.TraceError, "not canonical"): + self._read_sealed_bundle(noncanonical) + + duplicate = Path(temp) / "duplicate" + shutil.copytree(baseline, duplicate) + manifest_path = duplicate / trace.MANIFEST_NAME + data = manifest_path.read_text(encoding="ascii") + manifest_path.write_text(data.replace("{", '{"trace_format":"dsv41-trace",', 1), encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "duplicate JSON key"): + self._read_sealed_bundle(duplicate) + + truncated = Path(temp) / "truncated" + shutil.copytree(baseline, truncated) + events_path = truncated / trace.EVENTS_NAME + events_path.write_bytes(events_path.read_bytes()[:-1]) + with self.assertRaisesRegex(trace.TraceError, "truncated"): + self._read_sealed_bundle(truncated) + + missing = Path(temp) / "missing" + shutil.copytree(baseline, missing) + next((missing / trace.BLOBS_DIR).iterdir()).unlink() + with self.assertRaises(trace.TraceError): + self._read_sealed_bundle(missing) + + unsigned = Path(temp) / "unsigned" + shutil.copytree(baseline, unsigned) + (unsigned / trace.SIGNATURE_NAME).unlink() + with self.assertRaisesRegex(trace.TraceError, "bundle-signature"): + self._read_sealed_bundle(unsigned) + + for field, value in ( + ("namespace", "wrong-namespace"), + ("principal", "other-principal"), + ("format", "other-format"), + ("version", 2)): + with self.subTest(envelope_field=field): + root = Path(temp) / f"envelope-{field}" + shutil.copytree(baseline, root) + envelope_path = root / trace.SIGNATURE_NAME + envelope = json.loads(envelope_path.read_text(encoding="ascii")) + envelope[field] = value + envelope_path.write_text( + json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + with self.assertRaises(trace.TraceError): + self._read_sealed_bundle(root) + + def test_bundle_cannot_supply_signature_trust_inputs(self) -> None: + for field in ("signer_public_key", "signer_principal", "verifier_path", "allowed_signers"): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + record = manifest() + record[field] = "attacker-controlled" + with trace.TraceBundleWriter(root, record) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, f"unexpected {field}"): + trace.TraceBundle(root) + + def test_seal_rejects_hard_links_and_post_verify_swaps(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + self._seal_test_bundle(root) + bundle = self._read_sealed_bundle(root) + event = bundle.events[0] + blob = root / event["blob"] + replacement = root / "replacement" + replacement.write_bytes(blob.read_bytes()) + os.replace(replacement, blob) + with self.assertRaisesRegex(trace.TraceError, "changed after signature verification"): + bundle.read_blob(event) + + if os.name != "nt": + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + blob = next((root / trace.BLOBS_DIR).iterdir()) + os.link(blob, Path(temp) / "hard-link") + with self.assertRaisesRegex(trace.TraceError, "hard linked"): + self._seal_test_bundle(root) + + def test_serialization_preserves_float_bits_and_hashes(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + bits = (0x3F800000, 0x80000000, 0x7FC12345, 0) + data = struct.pack(" None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: + add_required_events(writer) + right_bundle = trace.TraceBundle(right) + expert = next(item for item in right_bundle.events if item["component"] == "expert.ids") + mutated = struct.pack( + " None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: + add_required_events(writer) + events_path = right / trace.EVENTS_NAME + events = [json.loads(line) for line in events_path.read_text(encoding="ascii").splitlines()] + for layer, element in ((0, 8), (1, 2)): + event = next(item for item in events if ( + item["component"] == "expert.ids" and + item["phase"] == "prefill" and + item["layer"] == layer + )) + values = list(struct.unpack(" None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + model = root / "model.gguf" + prompt = root / "prompt.txt" + exporter = root / "exporter" + output = root / "trace" + model.write_bytes(b"model") + prompt.write_bytes(b"line with trailing newline\n") + args = Namespace( + model=model, + prompt=prompt, + context=32768, + decode_steps=8, + batch=2048, + ubatch=trace.ADMITTED_UBATCH, + device="ROCm0", + gpu_layers=99, + expert_cache_slots=trace.REQUIRED_EXPERT_SLOTS, + expert_cache_mib=trace.REQUIRED_EXPERT_CACHE_MIB, + ) + command = run_llama.build_command(args, exporter, output) + self.assertEqual(command[command.index("-bf") + 1], str(prompt.resolve())) + self.assertEqual(command[command.index("--device") + 1], "ROCm0") + self.assertEqual(command[command.index("--load-mode") + 1], "none") + self.assertNotIn("-f", command) + + def test_rejects_unadmitted_expert_cache_configuration(self) -> None: + invalid = Namespace( + batch=trace.ADMITTED_BATCH, + ubatch=512, + device="ROCm0", + gpu_layers=99, + expert_cache_slots=8, + expert_cache_mib=4096, + ) + with self.assertRaisesRegex(preflight.PreflightError, "admitted ubatch 32"): + run_llama.validate_runtime_config(invalid) + + invalid.ubatch = trace.ADMITTED_UBATCH + with self.assertRaisesRegex(preflight.PreflightError, "192 expert cache slots"): + run_llama.validate_runtime_config(invalid) + + invalid.expert_cache_slots = trace.REQUIRED_EXPERT_SLOTS + with self.assertRaisesRegex(preflight.PreflightError, "76441190400 expert cache bytes"): + run_llama.validate_runtime_config(invalid) + + valid = Namespace( + batch=trace.ADMITTED_BATCH, + ubatch=trace.ADMITTED_UBATCH, + device="ROCm0", + gpu_layers=99, + expert_cache_slots=trace.REQUIRED_EXPERT_SLOTS, + expert_cache_mib=trace.REQUIRED_EXPERT_CACHE_MIB, + ) + run_llama.validate_runtime_config(valid) + self.assertEqual(trace.REQUIRED_EXPERT_CACHE_BYTES, 76_441_190_400) + self.assertEqual(trace.REQUIRED_EXPERT_CACHE_MIB, 72_900) + + def test_accelerator_attestation_rejects_wrong_missing_and_spoofed_architecture(self) -> None: + valid = dict(ACCELERATOR_ATTESTATION) + self.assertEqual( + run_llama.validate_accelerator_attestation(valid), + valid, + ) + for key, value, message in ( + ("architecture", "gfx1100", "architecture mismatch"), + ("architecture", None, "fields are invalid"), + ("backend_device", "ROCm1", "backend_device mismatch"), + ("gfx_target_version", 110500, "gfx_target_version mismatch"), + ("source", "environment", "source mismatch")): + invalid = dict(valid) + if value is None: + del invalid[key] + else: + invalid[key] = value + with self.assertRaisesRegex(preflight.PreflightError, message): + run_llama.validate_accelerator_attestation(invalid) + + spoofed = dict(valid) + spoofed["gfx_target_version"] = 110500 + spoofed["architecture"] = "gfx1151" + with self.assertRaisesRegex(preflight.PreflightError, "gfx_target_version mismatch"): + run_llama.validate_accelerator_attestation(spoofed) + + def test_accelerator_query_fails_closed(self) -> None: + valid_result = run_llama.subprocess.CompletedProcess( + ["exporter"], 0, json.dumps(ACCELERATOR_ATTESTATION), "") + with mock.patch.object(run_llama.subprocess, "run", return_value=valid_result): + self.assertEqual( + run_llama.query_accelerator_attestation(Path("/exporter"), "ROCm0"), + ACCELERATOR_ATTESTATION, + ) + + failed_result = run_llama.subprocess.CompletedProcess(["exporter"], 1, "", "query failed") + with mock.patch.object(run_llama.subprocess, "run", return_value=failed_result): + with self.assertRaisesRegex(preflight.PreflightError, "query failed"): + run_llama.query_accelerator_attestation(Path("/exporter"), "ROCm0") + + def test_metal_accelerator_attestation_is_runtime_specific(self) -> None: + valid = dict(METAL_ACCELERATOR_ATTESTATION) + self.assertEqual(run_ds4.validate_accelerator_attestation(valid), valid) + for mutation, message in ( + ({"runtime_kind": "strix-rocm"}, "runtime_kind mismatch"), + ({"platform": "linux"}, "platform mismatch"), + ({"backend": "ROCm"}, "backend mismatch"), + ({"source": "environment"}, "source mismatch"), + ({"pci_device_id": "0000:c1:00.0"}, "fields are invalid")): + invalid = dict(valid) + invalid.update(mutation) + with self.assertRaisesRegex(preflight.PreflightError, message): + run_ds4.validate_accelerator_attestation(invalid) + + def test_metal_accelerator_query_rejects_duplicate_keys(self) -> None: + duplicate = json.dumps(METAL_ACCELERATOR_ATTESTATION).replace( + '"backend": "Metal"', + '"backend": "Metal", "backend": "Metal"', + ) + device_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, duplicate.encode("utf-8"), b"") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, "run_exporter_command", + side_effect=[device_result, build_result]) as execute: + with self.assertRaisesRegex(preflight.PreflightError, "duplicate JSON key"): + run_ds4.query_accelerator_attestation( + Path("/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 2) + + def test_ds4_invalid_utf8_device_output_still_post_attests(self) -> None: + device_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[device_result, build_result], + ) as execute, self.assertRaisesRegex(preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual( + [call.args[0] for call in execute.call_args_list], + [ + ["/approved/exporter", "--dsv41-attest-device", "Metal0"], + ["/approved/exporter", "--dsv41-attest-build"], + ], + ) + + @unittest.skipUnless(os.name == "posix", "POSIX executable test") + def test_ds4_invalid_utf8_actual_process_still_runs_build_attestation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + marker = root / "build-attested" + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + "if test \"$1\" = --dsv41-attest-build; then\n" + f" printf build > '{marker}'\n" + " printf '{}'\n" + "else\n" + " printf '\\377'\n" + "fi\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + expected_build = fixture_runtime_build(policy) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + with mock.patch.object( + run_ds4, "validate_runtime_build_evidence", return_value=expected_build), self.assertRaisesRegex( + preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_accelerator_attestation( + exporter, + "Metal0", + exporter_identity=identity, + exporter_policy=policy, + expected_runtime_build=expected_build, + ) + self.assertEqual(marker.read_text(encoding="ascii"), "build") + + @unittest.skipUnless(os.name == "posix", "POSIX executable test") + def test_ds4_actual_invalid_utf8_and_post_build_failure_are_both_retained(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + policy, exporter = materialize_ds4_exporter_policy(root) + exporter.chmod(0o755) + exporter.write_text( + "#!/bin/sh\n" + "if test \"$1\" = --dsv41-attest-build; then\n" + " printf 'post-build failed' >&2\n" + " exit 9\n" + "fi\n" + "printf '\\377'\n", + encoding="ascii", + ) + exporter.chmod(0o555) + policy["executable_sha256"] = trace.sha256_file(exporter) + expected_build = fixture_runtime_build(policy) + with isolated_test_install_trust(): + identity = run_ds4.approved_executable_identity( + exporter, + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=policy["executable_path"], + expected_sha256=policy["executable_sha256"], + label="ds4 exporter", + ) + with self.assertRaisesRegex( + preflight.PreflightError, + "primary failure.*not valid UTF-8.*secondary post-invocation.*post-build failed", + ) as raised: + run_ds4.query_accelerator_attestation( + exporter, + "Metal0", + exporter_identity=identity, + exporter_policy=policy, + expected_runtime_build=expected_build, + ) + primary = raised.exception.__cause__ + self.assertIsInstance(primary, preflight.PreflightError) + self.assertIsInstance(primary.__cause__, UnicodeDecodeError) + self.assertIs(raised.exception.primary_error, primary) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["post-invocation-runtime-build-attestation"], + ) + + def test_ds4_invalid_utf8_build_attestation_is_nonrecursive(self) -> None: + invalid_result = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") + with mock.patch.object( + run_ds4, "run_exporter_command", return_value=invalid_result) as execute, self.assertRaisesRegex( + preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_runtime_build_attestation( + Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + ) + execute.assert_called_once() + + def test_ds4_main_unicode_error_still_post_attests(self) -> None: + runtime_trace = sys.modules["trace_format"] + primary = UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") + contained_error = runtime_trace.ExecutionIntegrityError( + "decode failed after contained execution", + primary_error=primary, + secondary_errors=[], + quiescence_proven=True, + ) + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[contained_error, build_result], + ) as execute, self.assertRaises(runtime_trace.ExecutionIntegrityError) as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertIs(raised.exception, contained_error) + self.assertEqual(execute.call_count, 2) + + def test_ds4_postflight_invalid_utf8_still_post_attests(self) -> None: + valid_device = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, json.dumps(METAL_ACCELERATOR_ATTESTATION).encode("utf-8"), b"") + invalid_device = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"\xff", b"") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[valid_device, build_result, invalid_device, build_result], + ) as execute: + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + with self.assertRaisesRegex(preflight.PreflightError, "not valid UTF-8"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 4) + self.assertEqual( + execute.call_args_list[-1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + + def test_ds4_unicode_and_post_attestation_failures_are_both_retained(self) -> None: + secondary = OSError("post-build launch failed") + invalid_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, b"\xff", b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[invalid_result, secondary], + ) as execute, self.assertRaisesRegex( + preflight.PreflightError, + "primary failure \\[PreflightError:.*not valid UTF-8.*secondary post-invocation.*" + "OSError: post-build launch failed") as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + capture_output=True, + decode_stdout_label="ds4 trace stdout", + decode_stderr_label="ds4 trace stderr", + ) + self.assertIsInstance(raised.exception.__cause__, preflight.PreflightError) + self.assertIsInstance(raised.exception.__cause__.__cause__, UnicodeDecodeError) + self.assertIs(raised.exception.primary_error, raised.exception.__cause__) + self.assertEqual( + [failure.component for failure in raised.exception.secondary_errors], + ["post-invocation-runtime-build-attestation"], + ) + self.assertEqual(execute.call_count, 2) + + def test_ds4_explicit_quiescence_false_always_blocks_post_attestation(self) -> None: + runtime_trace = sys.modules["trace_format"] + for component in ( + "process-tree-quiescence", + "windows-process-reap", + "windows-process-termination", + "windows-job-assignment", + "linux-helper-completion", + "linux-helper-protocol-shutdown", + "linux-helper-teardown", + "linux-root-pidfd-close", + "linux-namespace-pidfd-close", + "containment-helper-descriptor-close", + "executable-descriptor-close", + "runtime-descriptor-close", + "linux-process-fd-close:protocol", + "linux-process-fd-close:stdin", + "linux-process-fd-close:stdout", + "linux-process-fd-close:stderr", + "windows-process-handle-close", + "windows-thread-handle-close", + "containment-handle-close"): + with self.subTest(component=component): + primary = subprocess.TimeoutExpired(["exporter"], 7) + failure = runtime_trace._IntegrityFailure( + component, runtime_trace.TraceError("quiescence not proven")) + containment_error = runtime_trace.ExecutionIntegrityError( + "execution did not prove quiescence", + primary_error=primary, + secondary_errors=[failure], + quiescence_proven=False, + ) + with mock.patch.object( + run_ds4, "run_exporter_command", side_effect=containment_error) as execute, self.assertRaises( + runtime_trace.ExecutionIntegrityError) as raised: + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertIs(raised.exception, containment_error) + execute.assert_called_once() + + def test_ds4_accelerator_query_attests_after_launch_exceptions(self) -> None: + runtime_trace = sys.modules["trace_format"] + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + for primary in ( + OSError("device launch failed"), + subprocess.TimeoutExpired(["exporter"], 7), + ): + primary_error = runtime_trace.ExecutionIntegrityError( + "contained invocation failed", + primary_error=primary, + secondary_errors=[], + quiescence_proven=True, + ) + with self.subTest(error=type(primary).__name__), mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[primary_error, build_result], + ) as execute, self.assertRaises(runtime_trace.ExecutionIntegrityError): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 2) + self.assertEqual( + execute.call_args_list[0].args[0], + ["/approved/exporter", "--dsv41-attest-device", "Metal0"], + ) + self.assertEqual( + execute.call_args_list[1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + self.assertEqual( + execute.call_args_list[0].kwargs["timeout_seconds"], + run_ds4.EXPORTER_ATTESTATION_TIMEOUT_SECONDS, + ) + self.assertEqual( + execute.call_args_list[1].kwargs["timeout_seconds"], + run_ds4.EXPORTER_ATTESTATION_TIMEOUT_SECONDS, + ) + + def test_ds4_pre_spawn_failure_does_not_launch_post_attestation(self) -> None: + primary = OSError("process creation failed") + with mock.patch.object( + run_ds4, "run_exporter_command", side_effect=primary) as execute, self.assertRaises( + OSError) as raised: + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertIs(raised.exception, primary) + execute.assert_called_once() + + def test_ds4_accelerator_query_attests_after_nonzero_exit(self) -> None: + device_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 9, b"", b"device failed") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[device_result, build_result], + ) as execute, self.assertRaisesRegex(preflight.PreflightError, "device failed"): + run_ds4.query_accelerator_attestation( + Path("/approved/exporter"), + "Metal0", + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + ) + self.assertEqual(execute.call_count, 2) + + def test_ds4_invocation_reports_primary_and_post_attestation_failures(self) -> None: + runtime_trace = sys.modules["trace_format"] + primary_failures = ( + runtime_trace.ExecutionIntegrityError( + "contained invocation timed out", + primary_error=subprocess.TimeoutExpired(["exporter"], 7), + secondary_errors=[], + quiescence_proven=True, + ), + run_ds4.subprocess.CompletedProcess(["exporter"], 9, b"", b"device failed"), + ) + for primary_failure in primary_failures: + secondary_error = OSError("post-build launch failed") + with self.subTest(primary=type(primary_failure).__name__), mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[primary_failure, secondary_error], + ) as execute, self.assertRaisesRegex( + preflight.PreflightError, + "primary failure \\[(ExecutionIntegrityError|PreflightError):.*secondary post-invocation.*" + "OSError: post-build launch failed"): + run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--dsv41-attest-device", "Metal0"], + operation="selected accelerator query", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=7, + check=False, + ) + self.assertEqual(execute.call_count, 2) + self.assertEqual( + execute.call_args_list[1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + + def test_ds4_main_trace_result_waits_for_post_attestation(self) -> None: + trace_result = run_ds4.subprocess.CompletedProcess(["exporter"], 11, b"", b"trace failed") + build_result = run_ds4.subprocess.CompletedProcess( + ["exporter"], 0, trace.canonical_json(DS4_RUNTIME_BUILD).encode("utf-8"), b"") + with mock.patch.object( + run_ds4, + "run_exporter_command", + side_effect=[trace_result, build_result], + ) as execute: + result = run_ds4.run_exporter_with_post_attestation( + ["/approved/exporter", "--model", "/model.gguf"], + operation="ds4 trace execution", + exporter=Path("/approved/exporter"), + exporter_identity=object(), + exporter_policy=DS4_EXPORTER_POLICY, + expected_runtime_build=DS4_RUNTIME_BUILD, + timeout_seconds=run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + check=False, + ) + self.assertEqual(result.returncode, 11) + self.assertEqual(execute.call_count, 2) + self.assertEqual( + execute.call_args_list[1].args[0], + ["/approved/exporter", "--dsv41-attest-build"], + ) + self.assertEqual( + execute.call_args_list[0].kwargs["timeout_seconds"], + run_ds4.EXPORTER_TRACE_TIMEOUT_SECONDS, + ) + + def test_ds4_exporter_command_requires_bounded_timeout(self) -> None: + completed = run_ds4.subprocess.CompletedProcess(["exporter"], 0, b"", b"") + identity = object() + with mock.patch.object( + run_ds4, "run_approved_executable", return_value=(completed, identity)) as execute: + result = run_ds4.run_exporter_command( + ["/approved/exporter"], + exporter=Path("/approved/exporter"), + exporter_identity=identity, + exporter_policy=DS4_EXPORTER_POLICY, + timeout_seconds=17, + check=False, + ) + self.assertIs(result, completed) + self.assertEqual(execute.call_args.kwargs["timeout"], 17) + for timeout in (None, 0, -1, True): + kwargs = {} if timeout is None else {"timeout_seconds": timeout} + with self.subTest(timeout=timeout), self.assertRaisesRegex( + preflight.PreflightError, "timeout is invalid"): + run_ds4.run_exporter_command( + ["/approved/exporter"], + exporter=Path("/approved/exporter"), + exporter_identity=identity, + exporter_policy=DS4_EXPORTER_POLICY, + **kwargs, + ) + + def test_nvme_attestation_uses_mount_and_block_ancestry(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + xfs = root / "mnt" / "models" + btrfs = root / "home" + rotating = root / "rotating" + ram = root / "ram" + network = root / "network" + missing = root / "missing" + for directory in (xfs, btrfs, rotating, ram, network, missing): + directory.mkdir(parents=True) + (directory / "data").write_bytes(b"x") + + sys_root = root / "sys" + nvme1 = sys_root / "devices" / "pci" / "block" / "nvme1n1" + nvme0 = sys_root / "devices" / "pci" / "block" / "nvme0n1" + nvme0p3 = nvme0 / "nvme0n1p3" + sdb = sys_root / "devices" / "pci" / "block" / "sdb" + sdb1 = sdb / "sdb1" + for disk, rotational in ((nvme1, "0\n"), (nvme0, "0\n"), (sdb, "1\n")): + (disk / "queue").mkdir(parents=True) + (disk / "queue" / "rotational").write_text(rotational, encoding="ascii") + nvme0p3.mkdir() + sdb1.mkdir() + dev_block = sys_root / "dev" / "block" + dev_block.mkdir(parents=True) + (dev_block / "259:0").symlink_to(nvme1, target_is_directory=True) + (dev_block / "259:3").symlink_to(nvme0p3, target_is_directory=True) + (dev_block / "8:17").symlink_to(sdb1, target_is_directory=True) + class_block = sys_root / "class" / "block" + (class_block / "nvme0n1p3").mkdir(parents=True) + (class_block / "nvme0n1p3" / "dev").write_text("259:3\n", encoding="ascii") + + mountinfo = root / "mountinfo" + mountinfo.write_text( + f"1 0 259:0 / {xfs.resolve()} rw - xfs /dev/nvme1n1 rw\n" + f"2 0 0:35 /home {btrfs.resolve()} rw - btrfs /dev/nvme0n1p3[/home] rw\n" + f"3 0 8:17 / {rotating.resolve()} rw - ext4 /dev/sdb1 rw\n" + f"4 0 0:42 / {ram.resolve()} rw - tmpfs tmpfs rw\n" + f"5 0 0:43 / {network.resolve()} rw - nfs server:/share rw\n" + f"6 0 240:1 / {missing.resolve()} rw - ext4 /dev/missing rw\n", + encoding="ascii", + ) + + xfs_result = preflight.storage_attestation( + xfs / "data", + "xfs", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + self.assertEqual(xfs_result["filesystem_type"], "xfs") + self.assertEqual(xfs_result["nvme_device"], "nvme1n1") + + btrfs_result = preflight.storage_attestation( + btrfs / "new" / "trace", + "btrfs", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + self.assertEqual(btrfs_result["filesystem_type"], "btrfs") + self.assertEqual(btrfs_result["device_number"], "259:3") + self.assertEqual(btrfs_result["existing_path"], str(btrfs.resolve())) + self.assertEqual(btrfs_result["nvme_device"], "nvme0n1") + + for path, message in ( + (rotating / "data", "non-rotational"), + (ram / "data", "local block device"), + (network / "data", "local block device"), + (missing / "data", "cannot be resolved")): + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.storage_attestation( + path, + "invalid", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + + (xfs / "escape").symlink_to(ram, target_is_directory=True) + with self.assertRaisesRegex(preflight.PreflightError, "local block device"): + preflight.storage_attestation( + xfs / "escape" / "data", + "symlink escape", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + with self.assertRaisesRegex(preflight.PreflightError, "/mnt/bigspace"): + preflight.storage_attestation( + Path("/mnt/bigspace/model.gguf"), + "forbidden", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + ) + forbidden = root / "forbidden" + forbidden.mkdir() + (forbidden / "escape").symlink_to(xfs, target_is_directory=True) + with self.assertRaisesRegex(preflight.PreflightError, "must not use"): + preflight.storage_attestation( + forbidden / "escape" / "model.gguf", + "forbidden symlink", + mountinfo_path=mountinfo, + sys_dev_block_root=dev_block, + sys_class_block_root=class_block, + forbidden_root=forbidden, + ) + + def test_darwin_storage_and_host_preflight_are_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + for name in ("repo", "ds4", "tmp", "output"): + (root / name).mkdir() + model = root / "model.gguf" + prompt = root / "prompt.txt" + model.write_bytes(b"model") + prompt.write_bytes(b"prompt") + runner_executable = root / "python3" + runner_script = root / "repo" / "tools" / "deepseek-v41-trace" / "run_ds4.py" + exporter = root / "ds4-trace" + runner_script.parent.mkdir(parents=True) + runner_executable.write_bytes(b"python") + runner_script.write_bytes(b"runner") + exporter.write_bytes(b"exporter") + + def disk_info(path: Path) -> dict[str, object]: + return { + "MountPoint": str(root.resolve()), + "FilesystemType": "apfs", + "DeviceIdentifier": "disk3s1", + "ParentWholeDisk": "disk3", + "BusProtocol": "Apple Fabric", + "Internal": True, + "SolidState": True, + "VolumeNetwork": False, + "DiskImage": False, + } + + def command_text(*args: str) -> str: + commands = { + ("sysctl", "-n", "hw.memsize"): str(256 * 1024 * 1024 * 1024), + ("sysctl", "-n", "hw.model"): "Mac14,8", + ("sysctl", "-n", "kern.osproductversion"): "15.6", + ("sysctl", "-n", "vm.swapusage"): "total = 0.00M used = 0.00M free = 0.00M (encrypted)", + ("vm_stat",): ( + "Mach Virtual Memory Statistics: (page size of 16384 bytes)\n" + "Pages free: 1000000.\n" + "Pages inactive: 1000000.\n" + "Pages speculative: 1000000.\n" + ), + ("ps", "-axo", "pid=,ppid=,command="): "", + } + return commands[args] + + runner = dict(DS4_RUNNER_ATTESTATION) + runner["runner_executable"] = str(runner_executable.resolve()) + runner["runner_executable_sha256"] = trace.sha256_file(runner_executable) + runner["runner_script"] = str(runner_script.resolve()) + runner["runner_script_sha256"] = trace.sha256_file(runner_script) + runner["exporter_path"] = str(exporter.resolve()) + runner["exporter_sha256"] = trace.sha256_file(exporter) + runner["checkout_path"] = str((root / "ds4").resolve()) + with mock.patch.dict(preflight.os.environ, {"TMPDIR": str(root / "tmp")}, clear=True): + result = preflight.run_oracle_preflight( + model=model, + prompt=prompt, + output=root / "output", + repo=root / "repo", + checkout=root / "ds4", + busy_patterns=[], + accelerator=dict(METAL_ACCELERATOR_ATTESTATION), + runner=runner, + disk_info=disk_info, + command_text=command_text, + system="Darwin", + machine="arm64", + ) + self.assertEqual(result["runtime_kind"], "apple-metal") + self.assertEqual(result["storage"]["model"]["storage_kind"], "darwin-local-solid-state") + self.assertEqual(result["host"]["memory_bytes"], 256 * 1024 * 1024 * 1024) + + for mutation, message in ( + ({"Internal": False}, "internal non-rotational"), + ({"SolidState": False}, "internal non-rotational"), + ({"VolumeNetwork": True}, "local storage"), + ({"DiskImage": True}, "local storage"), + ({"BusProtocol": "Network"}, "NVMe-backed"), + ({"BusProtocol": "SATA"}, "NVMe-backed")): + def invalid_info(path: Path, mutation: dict[str, object] = mutation) -> dict[str, object]: + result = disk_info(path) + result.update(mutation) + return result + + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.darwin_storage_attestation( + model, + "model", + disk_info=invalid_info, + ) + + with self.assertRaisesRegex(preflight.PreflightError, "macOS on arm64"): + preflight.darwin_host_and_memory_audit( + command_text=command_text, + system="Linux", + machine="x86_64", + ) + + def test_darwin_storage_queries_the_containing_mount(self) -> None: + path = Path("/Users/test/model.gguf") + disk_info = { + "DeviceIdentifier": "disk3s5", + "ParentWholeDisk": "disk3", + "BusProtocol": "Apple Fabric", + "Internal": True, + "SolidState": True, + } + + def check_output(command, **_kwargs): + if command == ["df", "-P", str(path)]: + return ( + "Filesystem 512-blocks Used Available Capacity Mounted on\n" + "/dev/disk3s5 100 10 90 10% /System/Volumes/Data\n" + ) + self.assertEqual(command, ["diskutil", "info", "-plist", "/System/Volumes/Data"]) + return preflight.plistlib.dumps(disk_info) + + with mock.patch.object(preflight.subprocess, "check_output", side_effect=check_output): + result = preflight._diskutil_info(path) + self.assertEqual(result["_dsv41_mount_point"], "/System/Volumes/Data") + self.assertEqual({key: value for key, value in result.items() if not key.startswith("_")}, disk_info) + + def test_tmpdir_rejects_symlink_components(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + actual = root / "actual" + child = actual / "child" + child.mkdir(parents=True) + link = root / "link" + link.symlink_to(actual, target_is_directory=True) + for path in (link, Path(str(link) + "/"), link / ".", link / "child"): + with self.subTest(path=path), self.assertRaisesRegex( + preflight.PreflightError, "symlink"): + preflight.require_safe_tmpdir_path(path) + with self.assertRaisesRegex(preflight.PreflightError, "must not use /mnt/bigspace"): + preflight.require_safe_tmpdir_path(Path("/mnt/bigspace/escape")) + + def test_full_preflights_reject_unusable_lexical_tmpdir(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + (root / "good" / "tmp").mkdir(parents=True) + unusable = root / "good" / "missing" / ".." / "tmp" + actual = root / "actual" + actual.mkdir() + link = root / "link" + link.symlink_to(actual, target_is_directory=True) + home = root / "home" + (home / "tmp").mkdir(parents=True) + self.assertFalse(unusable.is_dir()) + self.assertFalse(os.access(unusable, os.W_OK | os.X_OK)) + self.assertTrue(link.is_symlink()) + + def strix_storage(_path: Path, label: str) -> dict[str, object]: + if label == "temporary directory": + raise AssertionError("unusable TMPDIR reached storage attestation") + return storage_record("/home/test") + + def oracle_storage( + _path: Path, + label: str, + **_kwargs: object) -> dict[str, object]: + if label == "temporary directory": + raise AssertionError("unusable TMPDIR reached storage attestation") + return metal_storage_record("/Users/oracle/test") + + def assert_rejected(tmpdir: Path, message: str) -> None: + with mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(tmpdir), "HOME": str(home)}, + clear=True), mock.patch.object( + preflight, + "storage_attestation", + side_effect=strix_storage): + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + with mock.patch.dict( + preflight.os.environ, + {"TMPDIR": str(tmpdir), "HOME": str(home)}, + clear=True), mock.patch.object( + preflight, + "darwin_storage_attestation", + side_effect=oracle_storage): + with self.assertRaisesRegex(preflight.PreflightError, message): + preflight.run_oracle_preflight( + model=Path("/Users/oracle/model.gguf"), + prompt=Path("/Users/oracle/prompt.txt"), + output=Path("/Users/oracle/trace"), + repo=Path("/Users/oracle/repo"), + checkout=Path("/Users/oracle/ds4"), + busy_patterns=[], + accelerator={}, + runner={}, + ) + + for tmpdir, message in ( + (unusable, "original lexical path"), + (link, "symlink")): + with self.subTest(tmpdir=tmpdir): + assert_rejected(tmpdir, message) + + literal_parent = root / "~" + literal_target = root / "literal" + (literal_target / "tmp").mkdir(parents=True) + literal_parent.symlink_to(literal_target, target_is_directory=True) + previous_cwd = Path.cwd() + try: + os.chdir(root) + literal_home = Path("~/tmp") + self.assertTrue(literal_home.is_dir()) + self.assertTrue(os.access(literal_home, os.W_OK | os.X_OK)) + self.assertNotEqual(literal_home.absolute(), literal_home.expanduser().absolute()) + assert_rejected(literal_home, "absolute literal path") + finally: + os.chdir(previous_cwd) + + model = root / "model.gguf" + prompt = root / "prompt.txt" + output = root / "output" + repo = root / "repo" + model.write_bytes(b"model") + prompt.write_bytes(b"prompt") + output.mkdir() + repo.mkdir() + valid_tmpdir = root / "good" / "tmp" + + def valid_strix_storage(path: Path, _label: str) -> dict[str, object]: + return storage_record(str(path.resolve())) + + with mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(valid_tmpdir)}, + clear=True), mock.patch.object( + preflight, "storage_attestation", side_effect=valid_strix_storage), mock.patch.object( + preflight, "swap_audit", return_value={"enabled": False, "entries": []}), mock.patch.object( + preflight, "watchdog_audit", return_value=copy.deepcopy( + AUDIT_RECORDS["watchdog"]["data"])), mock.patch.object( + preflight, "matching_workloads", return_value=[]), mock.patch.object( + preflight, "memory_audit", return_value=copy.deepcopy( + AUDIT_RECORDS["memory"]["data"])): + result = preflight.run_strix_preflight( + model=model, + prompt=prompt, + output=output, + repo=repo, + busy_patterns=[], + ) + self.assertEqual(result["storage"]["temporary_directory"]["resolved_path"], str(valid_tmpdir)) + + def test_preflight_requires_explicit_nvme_tmpdir(self) -> None: + with mock.patch.object( + preflight, + "storage_attestation", + return_value=storage_record("/home/test")): + with mock.patch.dict(preflight.os.environ, {"HIP_LAUNCH_BLOCKING": "1"}, clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "TMPDIR is required"): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + missing = root / "missing" + with mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(missing)}, + clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "existing writable directory"): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + actual = root / "actual" + (actual / "child").mkdir(parents=True) + link = root / "link" + link.symlink_to(actual, target_is_directory=True) + for path in (link, Path(str(link) + "/"), link / ".", link / "child"): + with self.subTest(path=path), mock.patch.dict( + preflight.os.environ, + {"HIP_LAUNCH_BLOCKING": "1", "TMPDIR": str(path)}, + clear=True): + with self.assertRaisesRegex(preflight.PreflightError, "must not be a symlink"): + preflight.run_strix_preflight( + model=Path("/home/model.gguf"), + prompt=Path("/home/prompt.txt"), + output=Path("/home/trace"), + repo=Path("/home/repo"), + busy_patterns=[], + ) + + def test_watchdog_lease_rejects_arbitrary_heartbeat_process(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + repo = root / "repo" + script = repo / "scripts" / "strix_memory_watchdog.py" + script.parent.mkdir(parents=True) + script.write_text("#!/usr/bin/env python3\n", encoding="ascii") + + procfs = root / "proc" + watchdog_pid = 123 + child_pid = 456 + current_pid = 789 + for pid in (watchdog_pid, child_pid, current_pid): + (procfs / str(pid)).mkdir(parents=True) + + def stat(pid: int, parent: int, start: int) -> str: + fields = ["S", str(parent), *(["0"] * 17), str(start)] + return f"{pid} (test) " + " ".join(fields) + "\n" + + (procfs / str(watchdog_pid) / "stat").write_text( + stat(watchdog_pid, 1, 1000), encoding="ascii") + (procfs / str(child_pid) / "stat").write_text( + stat(child_pid, watchdog_pid, 2000), encoding="ascii") + (procfs / str(current_pid) / "stat").write_text( + stat(current_pid, child_pid, 3000), encoding="ascii") + + watchdog_command = ( + b"python3\0" + str(script.resolve()).encode("ascii") + + b"\0--soft-gib\0" + b"116\0--emergency-gib\0" + b"118\0" + ) + child_command = b"python3\0run_matrix.py\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(watchdog_command) + (procfs / str(child_pid) / "cmdline").write_bytes(child_command) + (procfs / str(watchdog_pid) / "cwd").symlink_to(repo) + + heartbeat = root / "heartbeat.json" + audit = root / "watchdog.jsonl" + lease = root / "lease.json" + lease_id = "1" * 32 + heartbeat.write_text(json.dumps({ + "format": preflight.WATCHDOG_HEARTBEAT_FORMAT, + "version": preflight.WATCHDOG_VERSION, + "lease_id": lease_id, + "sequence": 1, + "state": "active", + "updated_at": "1970-01-01T00:01:40.000Z", + "updated_monotonic_ns": 1, + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "child_pid": child_pid, + "child_process_group_id": child_pid, + "sample": {}, + }), encoding="ascii") + child_argv = ["python3", "run_matrix.py"] + watchdog_events = json.loads(json.dumps(WATCHDOG_EVENTS)) + watchdog_events[1]["child_pid"] = child_pid + watchdog_events[1]["process_group_id"] = child_pid + watchdog_events[1]["command"] = child_argv + audit.write_text( + "".join(json.dumps(event) + "\n" for event in watchdog_events), + encoding="ascii", + ) + lease_record = { + "format": preflight.WATCHDOG_LEASE_FORMAT, + "version": preflight.WATCHDOG_VERSION, + "lease_id": lease_id, + "state": "active", + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "watchdog_command_sha256": preflight.sha256_bytes(watchdog_command), + "watchdog_script_path": str(script.resolve()), + "watchdog_script_sha256": preflight.sha256_bytes(script.read_bytes()), + "soft_bytes": preflight.SOFT_MEMORY_LIMIT, + "emergency_bytes": preflight.WATCHDOG_EMERGENCY_LIMIT, + "strict_ceiling_bytes": preflight.STRICT_MEMORY_LIMIT, + "procfs_root": "/proc", + "child_pid": child_pid, + "child_process_group_id": child_pid, + "command": child_argv, + "child_command_sha256": preflight.sha256_bytes(child_command), + "heartbeat_path": str(heartbeat), + "max_heartbeat_age_seconds": 5.0, + "audit_path": str(audit), + } + lease_record["child_command_sha256"] = preflight.sha256_bytes( + json.dumps(child_argv, ensure_ascii=True, separators=(",", ":")).encode("utf-8")) + lease.write_text(json.dumps(lease_record), encoding="ascii") + environment = { + preflight.WATCHDOG_LEASE_ENV: str(lease), + preflight.WATCHDOG_HEARTBEAT_ENV: str(heartbeat), + preflight.WATCHDOG_AUDIT_ENV: str(audit), + preflight.WATCHDOG_MAX_AGE_ENV: "5.0", + } + result = preflight.watchdog_audit( + repo, + environment=environment, + procfs_root=procfs, + current_pid=current_pid, + current_pgid=child_pid, + getpgid=lambda pid: child_pid, + now=100, + monotonic_ns=lambda: 1_000_000_001, + timeout_seconds=0, + ) + self.assertEqual(result["child_process_group_id"], child_pid) + + arbitrary = root / "arbitrary-heartbeat.py" + arbitrary.write_text("#!/usr/bin/env python3\n", encoding="ascii") + arbitrary_command = b"python3\0" + str(arbitrary.resolve()).encode("ascii") + b"\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(arbitrary_command) + lease_record["watchdog_command_sha256"] = preflight.sha256_bytes(arbitrary_command) + lease.write_text(json.dumps(lease_record), encoding="ascii") + with self.assertRaisesRegex(preflight.PreflightError, "candidate repository script"): + preflight.watchdog_audit( + repo, + environment=environment, + procfs_root=procfs, + current_pid=current_pid, + current_pgid=child_pid, + getpgid=lambda pid: child_pid, + now=100, + monotonic_ns=lambda: 1_000_000_001, + timeout_seconds=0, + ) + + def test_canonical_watchdog_validation_is_pinned_and_delegated(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + repo = root / "repo" + script = repo / "scripts" / "strix_memory_watchdog.py" + script.parent.mkdir(parents=True) + script.write_text("# fixture\n", encoding="ascii") + lease = root / "lease.json" + heartbeat = root / "heartbeat.json" + audit = root / "audit.jsonl" + lease.write_text("{}\n", encoding="ascii") + heartbeat.write_text( + json.dumps({"updated_at": "1970-01-01T00:00:01.000Z"}) + "\n", + encoding="ascii", + ) + audit.write_bytes(WATCHDOG_JSONL) + environment = { + preflight.WATCHDOG_LEASE_ENV: str(lease), + preflight.WATCHDOG_HEARTBEAT_ENV: str(heartbeat), + preflight.WATCHDOG_AUDIT_ENV: str(audit), + preflight.WATCHDOG_MAX_AGE_ENV: "5", + } + + class FakeLeaseError(RuntimeError): + pass + + class FakeWatchdog: + LeaseValidationError = FakeLeaseError + calls = [] + + @classmethod + def validate_active_lease(cls, lease_path: Path, **kwargs: object) -> dict[str, object]: + cls.calls.append((lease_path, kwargs)) + return { + **AUDIT_RECORDS["watchdog"]["data"], + "audit_path": str(audit), + "heartbeat_path": str(heartbeat), + "child_pid": 456, + } + + @staticmethod + def start_process_group_lease_guard(*args: object, **kwargs: object) -> None: + raise AssertionError("descendant validation must not start the direct-child guard") + + original_sha256 = preflight.WATCHDOG_SCRIPT_SHA256 + original_approved = dict(preflight.APPROVED_WATCHDOGS) + try: + preflight.WATCHDOG_SCRIPT_SHA256 = preflight.sha256_bytes(script.read_bytes()) + preflight.APPROVED_WATCHDOGS[preflight.WATCHDOG_SCRIPT_SHA256] = ( + preflight.WATCHDOG_REVISION) + result = preflight.watchdog_audit( + repo, + environment=environment, + procfs_root=Path("/proc"), + current_pid=789, + watchdog_module=FakeWatchdog, + monotonic=lambda: 2.0, + sleeper=lambda _: None, + ) + finally: + preflight.APPROVED_WATCHDOGS.clear() + preflight.APPROVED_WATCHDOGS.update(original_approved) + preflight.WATCHDOG_SCRIPT_SHA256 = original_sha256 + self.assertEqual(result["watchdog_revision"], preflight.WATCHDOG_REVISION) + self.assertEqual(len(FakeWatchdog.calls), 1) + kwargs = FakeWatchdog.calls[0][1] + self.assertEqual(kwargs["expected_soft_bytes"], trace.SOFT_MEMORY_LIMIT) + self.assertEqual(kwargs["expected_emergency_bytes"], trace.WATCHDOG_EMERGENCY_LIMIT) + self.assertEqual(kwargs["expected_procfs_root"], Path("/proc")) + + def test_approved_watchdog_is_exact(self) -> None: + expected = { + "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f": + "778db6f50eae04e6c232c69b9575bdbd0747962b", + } + self.assertEqual(preflight.APPROVED_WATCHDOGS, expected) + self.assertEqual(trace.APPROVED_WATCHDOGS, expected) + self.assertEqual(preflight.WATCHDOG_VERSION, 2) + self.assertEqual(trace.WATCHDOG_VERSION, 2) + + def test_canonical_watchdog_artifacts_embed_and_validate(self) -> None: + revision = preflight.WATCHDOG_REVISION + repository = Path(__file__).parents[1] + script = repository / "scripts" / "strix_memory_watchdog.py" + self.assertTrue(script.is_file()) + source = script.read_bytes() + self.assertEqual(preflight.sha256_bytes(source), preflight.WATCHDOG_SCRIPT_SHA256) + subprocess.run( + ["git", "merge-base", "--is-ancestor", revision, "HEAD"], + cwd=repository, + check=True, + ) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + repo = repository + watchdog = preflight._load_watchdog_module(script) + + lease_path = root / "watchdog.lease" + heartbeat_path = root / "watchdog.heartbeat" + audit_path = root / "watchdog.jsonl" + child_command = ["python3", "run_matrix.py"] + arguments = [ + "--soft-gib", "116", + "--emergency-gib", "118", + "--grace-seconds", "30", + "--sample-interval-seconds", "1", + "--heartbeat-max-age-seconds", "5", + "--lease-path", str(lease_path), + "--heartbeat-path", str(heartbeat_path), + "--audit-path", str(audit_path), + "--", + *child_command, + ] + config = watchdog.parse_args(arguments) + paths = config.validate() + now = datetime.now(timezone.utc).replace(microsecond=0) + monotonic_ns = 1_000_000_000 + procfs = root / "proc" + watchdog_pid = os.getpid() + guardian_pid = watchdog_pid + 100000 + child_pid = guardian_pid + 1 + + def proc_stat(pid: int, parent: int, group: int, start: int) -> str: + fields = ["S", str(parent), str(group), *(["0"] * 16), str(start)] + return f"{pid} (test) " + " ".join(fields) + "\n" + + for pid in (watchdog_pid, guardian_pid, child_pid): + (procfs / str(pid)).mkdir(parents=True) + (procfs / str(watchdog_pid) / "stat").write_text( + proc_stat(watchdog_pid, 1, watchdog_pid, 1000), encoding="ascii") + (procfs / str(guardian_pid) / "stat").write_text( + proc_stat(guardian_pid, watchdog_pid, guardian_pid, 2000), encoding="ascii") + (procfs / str(child_pid) / "stat").write_text( + proc_stat(child_pid, guardian_pid, guardian_pid, 3000), encoding="ascii") + watchdog_argv = [sys.executable, str(script), *arguments] + watchdog_cmdline = b"\0".join(os.fsencode(value) for value in watchdog_argv) + b"\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(watchdog_cmdline) + (procfs / str(watchdog_pid) / "exe").symlink_to(Path(sys.executable).resolve()) + (procfs / str(watchdog_pid) / "cwd").symlink_to(repo) + + class FakeProcess: + pid = guardian_pid + + @staticmethod + def poll() -> None: + return None + + @staticmethod + def wait(timeout: float | None = None) -> int: + del timeout + return 0 + + guardian = watchdog.GuardianProcess(FakeProcess(), child_pid, -1) + snapshot = watchdog.HostSnapshot( + 128 * 1024 * 1024 * 1024, + 64 * 1024 * 1024 * 1024, + (), + ) + canonical_finals = {} + for classification, exit_code in ( + ("procfs_error", watchdog.EXIT_PROCFS_ERROR), + ("signal_error", watchdog.EXIT_SIGNAL_ERROR)): + stream = io.StringIO() + final_logger = watchdog.AuditLogger(stream, wall_clock=lambda: now) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._kill_and_finish( + final_logger, + guardian, + snapshot, + snapshot.used_bytes, + classification, + exit_code, + f"test {classification}", + lambda _pid, _signal: "sigkill_sent", + ) + final_event = trace.strict_json_loads(stream.getvalue().splitlines()[-1]) + self.assertNotIn("error", final_event) + trace.validate_watchdog_event(final_event) + canonical_finals[classification] = final_event + + def fail_signal(_pid: int, _signal: int) -> str: + raise watchdog.ProcessGroupError("test signal failure") + + signal_stream = io.StringIO() + signal_logger = watchdog.AuditLogger(signal_stream, wall_clock=lambda: now) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._kill_and_finish( + signal_logger, + guardian, + snapshot, + snapshot.used_bytes, + "procfs_error", + watchdog.EXIT_PROCFS_ERROR, + "test signal failure", + fail_signal, + ) + signal_final = trace.strict_json_loads(signal_stream.getvalue().splitlines()[-1]) + self.assertEqual(signal_final["classification"], "signal_error") + self.assertIsInstance(signal_final["error"], str) + trace.validate_watchdog_event(signal_final) + canonical_finals["signal-error-detail"] = signal_final + + secondary_stream = io.StringIO() + secondary_logger = watchdog.AuditLogger(secondary_stream, wall_clock=lambda: now) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._emit_final( + secondary_logger, + "signal_error", + watchdog.EXIT_SIGNAL_ERROR, + "test secondary error", + snapshot, + snapshot.used_bytes, + guardian, + 0, + "signal_error", + "primary signal failure", + preserve_primary_on_artifact_error=True, + secondary_errors=[{"component": "audit", "detail": "secondary audit failure"}], + ) + secondary_final = trace.strict_json_loads(secondary_stream.getvalue().splitlines()[-1]) + trace.validate_watchdog_event(secondary_final) + canonical_finals["signal-error-secondary"] = secondary_final + + class TimeoutProcess(FakeProcess): + @staticmethod + def wait(timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(child_command, timeout) + + timeout_stream = io.StringIO() + timeout_logger = watchdog.AuditLogger(timeout_stream, wall_clock=lambda: now) + timeout_guardian = watchdog.GuardianProcess(TimeoutProcess(), child_pid, -1) + with mock.patch.object(watchdog.signal, "pthread_sigmask", return_value=set()): + watchdog._kill_and_finish( + timeout_logger, + timeout_guardian, + snapshot, + snapshot.used_bytes, + "procfs_error", + watchdog.EXIT_PROCFS_ERROR, + "test timeout", + lambda _pid, _signal: "sigkill_sent", + ) + timeout_final = trace.strict_json_loads(timeout_stream.getvalue().splitlines()[-1]) + self.assertEqual(timeout_final["classification"], "termination_timeout") + self.assertEqual(timeout_final["process_group_status"], "sigkill_timeout") + self.assertIsInstance(timeout_final["error"], str) + trace.validate_watchdog_event(timeout_final) + canonical_finals["termination_timeout"] = timeout_final + + logger = watchdog.AuditLogger(io.StringIO(), wall_clock=lambda: now) + logger.open_persistent(audit_path) + state = watchdog._state_fields( + snapshot, snapshot.used_bytes, None, None, "not_created", "none") + logger.emit( + "preflight", + **state, + soft_bytes=config.soft_bytes, + emergency_bytes=config.emergency_bytes, + strict_ceiling_bytes=watchdog.STRICT_CEILING_BYTES, + ) + manager = watchdog.LeaseManager( + config, + paths, + process_procfs_root=procfs, + wall_clock=lambda: now, + monotonic_ns=lambda: monotonic_ns, + ) + manager.start(guardian, logger) + logger.lease_manager = manager + state = watchdog._state_fields( + snapshot, snapshot.used_bytes, guardian, None, "active", "none") + logger.emit("child_started", **state, command=child_command) + logger.heartbeat(state) + fd_path = procfs / str(watchdog_pid) / "fd" + fd_path.mkdir() + (fd_path / str(logger.persistent_identity()["fd"])).symlink_to(audit_path) + + validated = watchdog.validate_active_lease( + lease_path, + expected_script_path=script, + expected_executable_path=Path(sys.executable), + expected_soft_bytes=trace.SOFT_MEMORY_LIMIT, + expected_emergency_bytes=trace.WATCHDOG_EMERGENCY_LIMIT, + expected_procfs_root=Path("/proc"), + expected_command=child_command, + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=5.0, + current_process_id=child_pid, + process_procfs_root=procfs, + monotonic_ns=lambda: monotonic_ns, + pidfd_open=lambda _: os.open("/dev/null", os.O_RDONLY), + ) + watchdog_audit = preflight._watchdog_audit_result( + validated, + watchdog_revision=revision, + lease_path=lease_path, + heartbeat_path=heartbeat_path, + audit_path=audit_path, + audit_event_count=2, + procfs_root=procfs, + ) + watchdog_audit["namespace_authority"] = { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": 9, + "host_procfs_root": "/proc", + "watchdog_pid": watchdog_pid, + "watchdog_process_group_id": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "watchdog_executable_path": str(Path(sys.executable).resolve()), + "watchdog_command_sha256": watchdog_audit["watchdog_command_sha256"], + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": guardian_pid, + } + created_unix = int(now.timestamp()) + audit = { + "created_unix": created_unix, + "runtime_kind": "strix-rocm", + "memory": copy.deepcopy(AUDIT_RECORDS["memory"]["data"]), + "swap": copy.deepcopy(AUDIT_RECORDS["swap"]["data"]), + "watchdog": watchdog_audit, + "environment": copy.deepcopy(AUDIT_RECORDS["memory"]["environment"]), + "storage": copy.deepcopy(STORAGE_ATTESTATION), + "storage_policy": copy.deepcopy(trace.NO_EXTERNAL_STATE_STORAGE), + "accelerator": copy.deepcopy(ACCELERATOR_ATTESTATION), + } + trace_root = root / "trace" + with trace.TraceBundleWriter(trace_root, manifest("llama.cpp")) as writer: + add_required_events(writer) + audit_sets = {} + for phase in ("pre", "post"): + paths = preflight.write_audits(root / f"{phase}-audits", audit) + preflight.seal_audits(paths) + audit_sets[phase] = paths + preflight.bind_embedded_audits(trace_root, audit_sets) + try: + trace.TraceBundle(trace_root) + for classification, final_event in canonical_finals.items(): + final_root = root / f"trace-{classification}" + with trace.TraceBundleWriter(final_root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events( + final_root, phase, [*WATCHDOG_EVENTS, final_event]) + trace.TraceBundle(final_root) + finally: + logger.close() + + def test_workload_scan_ignores_guarded_process_ancestry(self) -> None: + with tempfile.TemporaryDirectory() as temp: + procfs = Path(temp) + + def add_process(pid: int, parent: int, command: bytes) -> None: + process = procfs / str(pid) + process.mkdir() + process.joinpath("stat").write_text( + f"{pid} (test) S {parent} " + " ".join(["0"] * 18) + "\n", + encoding="ascii", + ) + process.joinpath("cmdline").write_bytes(command) + + add_process(90, 1, b"python3\0scripts/strix_memory_watchdog.py\0DeepSeek-V4.1\0") + add_process(100, 90, b"python3\0run_matrix.py\0--ds4-checkout\0/home/papa/src/ds4-v41\0") + add_process(200, 1, b"/tmp/ds4-v41-worker\0") + self.assertEqual( + preflight.matching_workloads( + ["ds4-v41", "DeepSeek-V4.1"], + procfs_root=procfs, + current_pid=100, + ), + [{"pid": 200, "command": "/tmp/ds4-v41-worker"}], + ) + + def test_rejects_cache_slot_id_space(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + writer = trace.TraceBundleWriter(root, manifest()) + with self.assertRaisesRegex(trace.TraceError, "original expert IDs"): + writer.add_event( + component="expert.ids", phase="prefill", step=0, token_start=0, token_count=1, + layer=0, dtype="i32", shape=[1], data=struct.pack(" None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + events_path = root / trace.EVENTS_NAME + events_path.write_bytes(events_path.read_bytes()[:-1]) + with self.assertRaisesRegex(trace.TraceError, "truncated"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + bundle = trace.TraceBundle(root) + event = bundle.events[0] + (root / event["blob"]).write_bytes(b"bad") + with self.assertRaisesRegex(trace.TraceError, "truncated blob|corrupt blob"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + bad_manifest = manifest() + bad_manifest["audits"]["pre"]["watchdog"] = "watchdog.json" + with trace.TraceBundleWriter(root, bad_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "audit reference"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest() + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + (root / trace_manifest["audits"]["pre"]["memory"]["path"]).unlink() + with self.assertRaisesRegex(trace.TraceError, "missing or not regular"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest() + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + (root / trace_manifest["audits"]["post"]["watchdog"]["path"]).unlink() + with self.assertRaisesRegex(trace.TraceError, "missing or not regular"): + trace.TraceBundle(root) + + def test_rejects_unpinned_ds4_revision(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["revision"] = "a" * 40 + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "ds4 revision"): + trace.TraceBundle(root) + + def test_rejects_unbound_runtime_build_identity(self) -> None: + cases = [] + + ds4_manifest = manifest("ds4") + ds4_manifest["build"]["path"] = "/Users/attacker/unrelated-exporter" + cases.append((ds4_manifest, "ds4 exporter runtime build path")) + + short_revision_manifest = manifest() + short_revision_manifest["revision"] = "a" * 9 + short_revision_manifest["candidate"]["revision"] = "a" * 9 + cases.append((short_revision_manifest, "candidate exporter approval revision")) + + revision_manifest = manifest() + revision_manifest["candidate"]["revision"] = "d" * 40 + cases.append((revision_manifest, "candidate exporter approval")) + + executable_manifest = manifest() + executable_manifest["candidate"]["executable_path"] = "/home/repo/build/bin/other-exporter" + cases.append((executable_manifest, "candidate exporter approval executable path")) + + library_manifest = manifest() + library_manifest["build"]["runtime_libraries"][0]["sha256"] = "e" * 64 + library_manifest["build"]["runtime_libraries_post"][0]["sha256"] = "e" * 64 + cases.append((library_manifest, "candidate exporter approval")) + + added_module_manifest = manifest() + added_module_manifest["build"]["runtime_module_monitor"]["project_additions"] = [ + {"path": "lib/libggml-injected.module"}] + cases.append((added_module_manifest, "runtime module addition during trace generation")) + + incomplete_monitor_manifest = manifest() + incomplete_monitor_manifest["build"]["runtime_module_monitor"]["checked_after_trace"] = False + cases.append((incomplete_monitor_manifest, "runtime module monitor did not complete")) + + library_path_manifest = manifest() + library_path_manifest["build"]["runtime_libraries"][0]["path"] = ( + "/home/repo/build/bin/../substituted/libllama-common.so") + library_path_manifest["build"]["runtime_libraries_post"][0]["path"] = ( + "/home/repo/build/bin/../substituted/libllama-common.so") + cases.append((library_path_manifest, "runtime library path is not canonical")) + + external_library_manifest = manifest() + external_library_manifest["build"]["runtime_libraries"][0]["path"] = ( + "/Users/attacker/libggml-injected.dylib") + external_library_manifest["build"]["runtime_libraries"].sort(key=lambda item: item["path"]) + external_library_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + external_library_manifest["build"]["runtime_libraries"]) + cases.append((external_library_manifest, "outside the exporter runtime directory")) + + omitted_library_manifest = manifest() + omitted_library_manifest["build"]["runtime_libraries"] = [ + library + for library in omitted_library_manifest["build"]["runtime_libraries"] + if library["role"] != "selected-backend" + ] + omitted_library_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + omitted_library_manifest["build"]["runtime_libraries"]) + cases.append((omitted_library_manifest, "candidate exporter approval receipt")) + + duplicate_path_manifest = manifest() + duplicate_path_manifest["build"]["runtime_libraries"][1]["path"] = ( + duplicate_path_manifest["build"]["runtime_libraries"][0]["path"]) + duplicate_path_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_path_manifest["build"]["runtime_libraries"]) + cases.append((duplicate_path_manifest, "runtime library path is duplicated")) + + duplicate_role_manifest = manifest() + duplicate_role_manifest["build"]["runtime_libraries"][1]["role"] = ( + duplicate_role_manifest["build"]["runtime_libraries"][0]["role"]) + duplicate_role_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_role_manifest["build"]["runtime_libraries"]) + cases.append((duplicate_role_manifest, "runtime library role is invalid")) + + unknown_component_manifest = manifest() + unknown_component_manifest["build"]["runtime_libraries"][0]["component"] = "ggml-injected" + unknown_component_manifest["build"]["runtime_libraries_post"][0]["component"] = "ggml-injected" + cases.append((unknown_component_manifest, "candidate exporter approval runtime receipt component")) + + revision_library_manifest = manifest() + revision_library = next( + library + for library in revision_library_manifest["build"]["runtime_libraries"] + if library["role"] == "build-info") + revision_library["revision"] = "b" * 40 + next( + library + for library in revision_library_manifest["build"]["runtime_libraries_post"] + if library["role"] == "build-info")["revision"] = "b" * 40 + cases.append((revision_library_manifest, "candidate exporter approval runtime receipt revision")) + + unexpected_revision_manifest = manifest() + unexpected_revision = next( + library + for library in unexpected_revision_manifest["build"]["runtime_libraries"] + if library["role"].startswith("runtime:")) + unexpected_revision["revision"] = "a" * 40 + next( + library + for library in unexpected_revision_manifest["build"]["runtime_libraries_post"] + if library["role"].startswith("runtime:"))["revision"] = "a" * 40 + cases.append((unexpected_revision_manifest, "candidate exporter approval runtime receipt revision")) + + for trace_manifest, message in cases: + with self.subTest(message=message), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_rejects_unattested_accelerator_identity(self) -> None: + for key, value, message in ( + ("architecture", "gfx1100", "architecture mismatch"), + ("gfx_target_version", 110500, "gfx_target_version mismatch"), + ("pci_device_id", "ROCm0", "PCI identity")): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest() + trace_manifest["accelerator"][key] = value + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_accepts_truthful_metal_vs_strix_bundles(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "ds4" + right = Path(temp) / "llama" + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: + add_required_events(writer) + left_bundle = trace.TraceBundle(left) + right_bundle = trace.TraceBundle(right) + self.assertNotEqual( + left_bundle.manifest["accelerator"]["architecture"], + right_bundle.manifest["accelerator"]["architecture"], + ) + result = trace.report(left_bundle, right_bundle) + self.assertEqual(result["status"], "TARGET PASS") + + def test_rejects_cross_runtime_attestation_substitution(self) -> None: + for runtime, accelerator, message in ( + ("ds4", ACCELERATOR_ATTESTATION, "runtime profile"), + ("llama.cpp", METAL_ACCELERATOR_ATTESTATION, "llama.cpp accelerator attestation fields")): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest(runtime) + trace_manifest["accelerator"] = dict(accelerator) + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + for runtime, records, replacement, message in ( + ( + "ds4", + DS4_AUDIT_RECORDS, + storage_record("/mnt/models/model.gguf"), + "ds4 storage attestation fields", + ), + ( + "llama.cpp", + AUDIT_RECORDS, + metal_storage_record("/Users/oracle/model.gguf"), + "llama.cpp storage attestation fields", + )): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest(runtime)) as writer: + add_required_events(writer) + record = json.loads(json.dumps(records["memory"])) + record["storage"]["model"] = replacement + replace_audit_record(root, "pre", "memory", record) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_rejects_ds4_accelerator_audit_removal(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["memory"])) + del record["accelerator"] + replace_audit_record(root, phase, "memory", record) + with self.assertRaisesRegex(trace.TraceError, "missing accelerator"): + trace.TraceBundle(root) + + def test_rejects_ds4_sata_storage_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["memory"])) + record["storage"]["model"]["bus_protocol"] = "SATA" + replace_audit_record(root, phase, "memory", record) + with self.assertRaisesRegex(trace.TraceError, "not NVMe-backed"): + trace.TraceBundle(root) + + def test_rejects_cross_runtime_host_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("llama.cpp") + trace_manifest["host"] = dict(DS4_HOST_ATTESTATION) + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "unexpected host"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["host"]["runtime_kind"] = "strix-rocm" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "host runtime_kind mismatch"): + trace.TraceBundle(root) + + def test_rejects_storage_audit_path_substitution(self) -> None: + for runtime, records, different in ( + ("llama.cpp", AUDIT_RECORDS, "/mnt/models/different.gguf"), + ("ds4", DS4_AUDIT_RECORDS, "/Users/oracle/different.gguf")): + with self.subTest(runtime=runtime), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest(runtime)) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(records["memory"])) + record["storage"]["model"]["resolved_path"] = different + record["storage"]["model"]["existing_path"] = different + replace_audit_record(root, phase, "memory", record) + with self.assertRaisesRegex(trace.TraceError, "model path differs from the manifest"): + trace.TraceBundle(root) + + def test_rejects_duplicate_and_unknown_runtime_attestation_keys(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + data = manifest_path.read_text(encoding="ascii") + data = data.replace( + '"runtime_kind":"apple-metal"', + '"runtime_kind":"apple-metal","runtime_kind":"apple-metal"', + 1, + ) + manifest_path.write_text(data, encoding="ascii") + with self.assertRaisesRegex(trace.TraceError, "duplicate JSON key"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["accelerator"]["runtime_kind"] = "unknown" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "runtime profile"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + del trace_manifest["accelerator"]["runtime_kind"] + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "runtime profile"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["unknown_top_level"] = True + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "unexpected unknown_top_level"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["config"]["unvalidated_mode"] = "unsafe" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "unexpected unvalidated_mode"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["environment"]["system_info"] = "Linux test system" + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "environment is not macOS"): + trace.TraceBundle(root) + + def test_native_complete_manifest_writer_validates(self) -> None: + production_binary = Path(os.environ.get( + "DSV41_NATIVE_TRACE_BINARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "llama-deepseek-v41-trace", + )) + manifest_binary = Path(os.environ.get( + "DSV41_NATIVE_MANIFEST_BINARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "test-deepseek41-trace-manifest", + )) + if not production_binary.is_file() or not manifest_binary.is_file(): + self.skipTest("native trace exporter and manifest harness are not built") + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace with space" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + manifest_path = root / trace.MANIFEST_NAME + fixture = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + writer_input = { + key: fixture[key] + for key in ("model", "prompt", "audits", "expected", "event_count") + } + input_path = Path(temp) / "manifest-input.json" + input_path.write_text( + json.dumps(writer_input, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + manifest_path.unlink() + command = [ + str(manifest_binary.resolve()), + "--write-test-manifest", + str(input_path), + str(manifest_path), + ] + subprocess.run(command, check=True) + native = trace.strict_json_loads(manifest_path.read_text(encoding="ascii")) + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=Path(__file__).parents[1], + text=True, + ).strip() + version = subprocess.run( + [str(production_binary.resolve()), "--version"], + check=True, + capture_output=True, + text=True, + ) + self.assertIn(f"commit {revision}", version.stdout) + self.assertEqual(native["revision"], revision) + self.assertEqual(native["build"]["path"], str(manifest_binary.resolve())) + self.assertIn("test-only manifest harness", native["build"]["info"]) + self.assertEqual(native["accelerator"]["runtime_kind"], "native-test") + self.assertEqual(native["accelerator"]["device_type"], "cpu") + self.assertIs(native["accelerator"]["test_only"], True) + components = native["build"]["runtime_profile"]["components"] + self.assertEqual(components, sorted(components)) + self.assertEqual( + native["build"]["runtime_libraries_post"], + native["build"]["runtime_libraries"], + ) + self.assertEqual( + native["build"]["runtime_module_monitor"], + { + "mechanism": "dyld-add-image" if sys.platform == "darwin" else "pre-post-snapshot", + "checked_after_trace": True, + "project_additions": [], + }, + ) + self.assertEqual( + {library["component"] for library in native["build"]["runtime_libraries"]}, + set(components), + ) + self.assertEqual( + len({library["role"] for library in native["build"]["runtime_libraries"]}), + len(components), + ) + self.assertEqual( + [library["path"] for library in native["build"]["runtime_libraries"]], + sorted(library["path"] for library in native["build"]["runtime_libraries"]), + ) + for library in native["build"]["runtime_libraries"]: + expected_revision = ( + revision + if library["component"] in {"llama-common", "ggml-base"} + else None + ) + self.assertEqual(library["revision"], expected_revision) + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": revision, + "profile": native["build"]["runtime_profile"]["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in native["build"]["runtime_libraries"] + ], + key=lambda item: item["component"], + ), + } + self.assertEqual( + native["build"]["runtime_receipt_sha256"], + trace.sha256_bytes(trace.canonical_json(receipt).encode("ascii")), + ) + self.assertIsInstance(native["environment"]["command"], str) + self.assertEqual(json.loads(native["environment"]["command"]), command) + self.assertIs(native["config"]["flash_attention"], False) + self.assertEqual(native["config"]["runtime_kind"], "native-test") + self.assertEqual(native["config"]["device_type"], "cpu") + self.assertIs(native["config"]["test_only"], True) + self.assertEqual(native["storage_policy"], trace.NO_EXTERNAL_STATE_STORAGE) + attestation = fixture["candidate"] + attestation["revision"] = revision + attestation["executable_path"] = str(manifest_binary.resolve()) + attestation["executable_sha256"] = trace.sha256_file(manifest_binary) + with self.assertRaisesRegex(run_llama.PreflightError, "test-only manifest harness"): + run_llama.bind_candidate_attestation( + root, + attestation, + native["accelerator"], + manifest_binary, + trace.sha256_file(manifest_binary), + { + "runtime_profile": native["build"]["runtime_profile"], + "runtime_receipt": receipt, + }, + ) + with self.assertRaisesRegex(trace.TraceError, "execution authorization is missing"): + trace.seal_bundle( + root, + private_key=self.signing_key, + principal=self.signer_principal, + expected_lane=trace.CANDIDATE_LANE, + expected_challenge=TEST_CHALLENGE, + expected_run_id=TEST_RUN_IDS["llama.cpp"], + trusted_signers=self.test_signers, + ssh_keygen=self.ssh_keygen, + ) + + for protected_field in ( + "accelerator", "authorization", "build", "candidate", "comparison", "config", + "environment", "paths", "revision", "runtime", "storage_policy"): + protected_input = dict(writer_input) + protected_input[protected_field] = fixture.get(protected_field, {}) + input_path.write_text( + json.dumps(protected_input, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + rejected = subprocess.run(command, check=False, capture_output=True, text=True) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn(f"unexpected field: {protected_field}", rejected.stderr) + + for option in ("--dsv41-manifest-writer-probe", "--dsv41-runtime-module-path-probe"): + rejected = subprocess.run( + [str(production_binary.resolve()), option], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + + rejected = subprocess.run( + [str(production_binary.resolve()), "--dsv41-attest-device", "CPU"], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("selected execution device must be ROCm0", rejected.stderr) + + def test_runtime_build_validator_rejects_closure_substitution(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + binary_directory = root / "build" / "bin" + library_directory = root / "build" / "lib" + binary_directory.mkdir(parents=True) + library_directory.mkdir() + exporter = binary_directory / "llama-deepseek-v41-trace" + exporter.write_bytes(b"exporter") + records = [] + for component, name, role, content in ( + ("ggml", "libggml.so", "runtime:ggml", b"ggml"), + ("ggml-base", "libggml-base.so", "ggml", b"base"), + ("ggml-blas", "libggml-blas.so", "runtime:ggml-blas", b"blas"), + ("ggml-hip", "libggml-hip.so", "selected-backend", b"backend"), + ("llama", "libllama.so", "llama", b"llama"), + ("llama-common", "libllama-common.so", "build-info", b"build")): + path = library_directory / name + path.write_bytes(content) + records.append({ + "component": component, + "filename": name, + "path": str(path.resolve()), + "sha256": trace.sha256_file(path), + "role": role, + "revision": "a" * 40 if component in {"llama-common", "ggml-base"} else None, + }) + records.sort(key=lambda record: record["path"]) + components = sorted(record["component"] for record in records) + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "a" * 40, + "profile": "sibling-lib", + "components": sorted( + [ + { + "component": record["component"], + "filename": record["filename"], + "sha256": record["sha256"], + "revision": record["revision"], + } + for record in records + ], + key=lambda item: item["component"], + ), + } + build_manifest = { + "revision": "a" * 40, + "build": { + "path": str(exporter.resolve()), + "sha256": trace.sha256_file(exporter), + "info": "test", + "runtime_profile": { + "name": "sibling-lib", + "components": components, + "selected_backend_component": "ggml-hip", + }, + "runtime_receipt_sha256": trace.sha256_bytes( + trace.canonical_json(receipt).encode("ascii")), + "runtime_libraries": records, + "runtime_libraries_post": copy.deepcopy(records), + "runtime_module_monitor": { + "mechanism": "pre-post-snapshot", + "checked_after_trace": True, + "project_additions": [], + }, + }, + } + approval = { + "runtime_profile": copy.deepcopy(build_manifest["build"]["runtime_profile"]), + "runtime_receipt": copy.deepcopy(receipt), + } + libraries_digest, receipt_digest = run_llama.validate_runtime_build( + build_manifest, + exporter=exporter, + exporter_sha256=trace.sha256_file(exporter), + candidate_revision="a" * 40, + approval=approval, + ) + self.assertEqual( + libraries_digest, + trace.sha256_bytes(trace.canonical_json({ + "pre": records, + "post": records, + }).encode("ascii")), + ) + self.assertEqual(receipt_digest, build_manifest["build"]["runtime_receipt_sha256"]) + + cases = [] + omitted = copy.deepcopy(build_manifest) + omitted["build"]["runtime_libraries"] = [ + library + for library in omitted["build"]["runtime_libraries"] + if library["role"] != "selected-backend" + ] + omitted["build"]["runtime_libraries_post"] = copy.deepcopy( + omitted["build"]["runtime_libraries"]) + cases.append((omitted, "set differs from the runtime profile")) + + changed_hash = copy.deepcopy(build_manifest) + changed_hash["build"]["runtime_libraries"][0]["sha256"] = "f" * 64 + changed_hash["build"]["runtime_libraries_post"][0]["sha256"] = "f" * 64 + cases.append((changed_hash, "SHA-256 mismatch")) + + changed_post = copy.deepcopy(build_manifest) + changed_post["build"]["runtime_libraries_post"][0]["sha256"] = "f" * 64 + cases.append((changed_post, "closure changed during trace generation")) + + added_module = copy.deepcopy(build_manifest) + added_module["build"]["runtime_module_monitor"]["project_additions"] = [ + {"path": "lib/libggml-injected.module"}] + cases.append((added_module, "runtime module addition during trace generation")) + + incomplete_monitor = copy.deepcopy(build_manifest) + incomplete_monitor["build"]["runtime_module_monitor"]["checked_after_trace"] = False + cases.append((incomplete_monitor, "runtime module monitor did not complete")) + + changed_revision = copy.deepcopy(build_manifest) + revision_record = next( + library + for library in changed_revision["build"]["runtime_libraries"] + if library["role"] == "build-info") + revision_record["revision"] = "b" * 40 + next( + library + for library in changed_revision["build"]["runtime_libraries_post"] + if library["role"] == "build-info")["revision"] = "b" * 40 + cases.append((changed_revision, "revision mismatch")) + + duplicate_role = copy.deepcopy(build_manifest) + role_records = duplicate_role["build"]["runtime_libraries"] + next(library for library in role_records if library["role"].startswith("runtime:"))["role"] = "llama" + duplicate_role["build"]["runtime_libraries_post"] = copy.deepcopy(role_records) + cases.append((duplicate_role, "role is invalid")) + + duplicate_component = copy.deepcopy(build_manifest) + duplicate_component["build"]["runtime_libraries"][1]["component"] = ( + duplicate_component["build"]["runtime_libraries"][0]["component"]) + duplicate_component["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_component["build"]["runtime_libraries"]) + cases.append((duplicate_component, "component is invalid")) + + selected_backend = copy.deepcopy(build_manifest) + selected_backend["build"]["runtime_profile"]["selected_backend_component"] = "ggml-blas" + cases.append((selected_backend, "selected backend component is not ggml-hip")) + + filename = copy.deepcopy(build_manifest) + filename["build"]["runtime_libraries"][0]["filename"] = "other.so" + filename["build"]["runtime_libraries_post"] = copy.deepcopy( + filename["build"]["runtime_libraries"]) + cases.append((filename, "path differs from the exact runtime profile")) + + receipt_digest = copy.deepcopy(build_manifest) + receipt_digest["build"]["runtime_receipt_sha256"] = "f" * 64 + cases.append((receipt_digest, "runtime receipt SHA-256 mismatch")) + + external = root / "external" / "libggml-blas.so" + external.parent.mkdir() + external.write_bytes(b"blas") + external_manifest = copy.deepcopy(build_manifest) + external_record = next( + library + for library in external_manifest["build"]["runtime_libraries"] + if library["component"] == "ggml-blas") + external_record["path"] = str(external.resolve()) + external_manifest["build"]["runtime_libraries"].sort(key=lambda record: record["path"]) + external_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + external_manifest["build"]["runtime_libraries"]) + cases.append((external_manifest, "path differs from the exact runtime profile")) + + catalogued_not_profile = copy.deepcopy(build_manifest) + injected = library_directory / "libggml-injected.so" + injected.write_bytes(b"injected") + catalogued_not_profile["build"]["runtime_libraries"].append({ + "component": "ggml-injected", + "filename": injected.name, + "path": str(injected.resolve()), + "sha256": trace.sha256_file(injected), + "role": "runtime:ggml-injected", + "revision": None, + }) + catalogued_not_profile["build"]["runtime_libraries"].sort(key=lambda record: record["path"]) + catalogued_not_profile["build"]["runtime_libraries_post"] = copy.deepcopy( + catalogued_not_profile["build"]["runtime_libraries"]) + cases.append((catalogued_not_profile, "component is invalid")) + + duplicate_path = copy.deepcopy(build_manifest) + duplicate_path["build"]["runtime_libraries"][1]["path"] = ( + duplicate_path["build"]["runtime_libraries"][0]["path"]) + duplicate_path["build"]["runtime_libraries_post"] = copy.deepcopy( + duplicate_path["build"]["runtime_libraries"]) + cases.append((duplicate_path, "duplicated or unsorted")) + + for candidate, message in cases: + with self.subTest(message=message), self.assertRaisesRegex( + run_llama.PreflightError, message): + run_llama.validate_runtime_build( + candidate, + exporter=exporter, + exporter_sha256=trace.sha256_file(exporter), + candidate_revision="a" * 40, + approval=approval, + ) + + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "loader injection test") + def test_native_rejects_injected_project_library(self) -> None: + binary = Path(os.environ.get( + "DSV41_NATIVE_TRACE_BINARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "llama-deepseek-v41-trace", + )) + injected = Path(os.environ.get( + "DSV41_NATIVE_INJECT_LIBRARY", + Path(__file__).parents[1] / "build-harness" / "bin" / "libggml-injected.module", + )) + if not binary.is_file() or not injected.is_file(): + self.skipTest("native trace exporter and injected test library are not built") + with tempfile.TemporaryDirectory() as temp: + external = Path(temp) / injected.name + shutil.copy2(injected, external) + variable = "DYLD_INSERT_LIBRARIES" if sys.platform == "darwin" else "LD_PRELOAD" + inside = binary.parent / "renamed-injected.module" + shutil.copy2(injected, inside) + try: + for name, injected_path in (("outside", external), ("renamed-inside", inside)): + with self.subTest(name=name): + environment = dict(os.environ) + environment[variable] = str(injected_path) + rejected = subprocess.run( + [str(binary.resolve()), "--version"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("forbids loader override", rejected.stderr) + finally: + inside.unlink(missing_ok=True) + + def test_python_runner_rejects_loader_overrides(self) -> None: + for variable in trace.FORBIDDEN_LOADER_ENVIRONMENT: + with self.subTest(variable=variable), mock.patch.dict( + os.environ, {variable: "/tmp/untrusted-runtime"}, clear=True): + with self.assertRaisesRegex(trace.TraceError, variable): + trace.reject_loader_overrides() + + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "source alias test") + def test_prompt_builder_result_becomes_strict_provenance(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + builder = root / "install" / "bin" / "llama-deepseek-v41-prompt-builder" + model = root / "model.gguf" + corpus = root / "corpus.txt" + source_root = Path(__file__).parents[1] + source_corpus = source_root / "tests" / "corpus" / "correctness-prose.txt" + source_alias = root / "source-alias" + output = root / "prompt.txt" + tmpdir = root / "tmp" + builder.parent.mkdir(parents=True) + builder.write_bytes(b"builder") + builder.chmod(0o755) + model.write_bytes(b"model") + shutil.copyfile(source_corpus, corpus) + source_alias.symlink_to(source_root, target_is_directory=True) + tmpdir.mkdir() + builder = builder.resolve() + model = model.resolve() + corpus = corpus.resolve() + output = output.resolve() + tmpdir = tmpdir.resolve() + builder_policy = fixture_prompt_builder_policy( + b"prompt", + builder_path=str(builder.resolve()), + builder_sha256=trace.sha256_file(builder), + source_root=str(source_alias), + ) + materialize_policy_runtime(builder_policy) + self.assertEqual(run_matrix.approved_source_root(builder_policy), source_root.resolve()) + self.assertEqual(run_llama.approved_source_root(builder_policy), source_root.resolve()) + _validated, builder_policy_sha256 = trace.prompt_builder_approval( + TEST_PROMPT_BUILDER_POLICY_ID, + policies={TEST_PROMPT_BUILDER_POLICY_ID: builder_policy}, + ) + with isolated_test_install_trust(): + builder_identity = run_matrix.approved_executable_identity( + builder, + install_root=builder_policy["install_root"], + expected_owner_uid=builder_policy["install_owner_uid"], + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + ) + + def run_builder(command, **_kwargs): + if "--dsv41-attest-build" in command: + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps(fixture_runtime_build(builder_policy)), + "", + ), + builder_identity, + ) + output.write_bytes(b"prompt") + return ( + subprocess.CompletedProcess( + command, + 0, + json.dumps({ + "target_tokens": 2, + "actual_tokens": 2, + "byte_count": 6, + "tokenizer": builder_policy["tokenizer"], + "runtime_build": fixture_runtime_build(builder_policy), + "temporary_directory": str(tmpdir.resolve()), + }), + "", + ), + builder_identity, + ) + + with isolated_test_install_trust(), mock.patch.dict( + os.environ, {"TMPDIR": str(tmpdir)}, clear=True), mock.patch.object( + run_matrix, "run_approved_executable", side_effect=run_builder), mock.patch.object( + sys, "stderr", io.StringIO()): + result = run_matrix.prepare_prompt( + builder=builder, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=builder_policy, + builder_policy_sha256=builder_policy_sha256, + model=model, + corpus=corpus, + source_corpus=source_corpus, + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + output=output, + context=3, + decode_steps=1, + ) + provenance_path = Path(result["provenance_path"]) + provenance = trace.strict_json_loads(provenance_path.read_text(encoding="ascii")) + self.assertEqual(provenance["source_root_lexical_path"], str(source_alias)) + self.assertEqual(provenance["source_root_resolved_path"], str(source_root.resolve())) + self.assertEqual(provenance["corpus_lexical_path"], str(source_corpus)) + self.assertEqual(provenance["corpus_resolved_path"], str(source_corpus.resolve())) + self.assertEqual( + set(provenance), + { + "format", "version", "corpus_name", "corpus_sha256", "corpus_path", + "corpus_lexical_path", "corpus_resolved_path", + "source_root_lexical_path", "source_root_resolved_path", + "model_sha256", "prompt_sha256", "prompt_byte_count", "context", + "decode_steps", "builder_approval_id", "builder_approval_sha256", + "builder_path", "builder_sha256", "builder_revision", + "builder_runtime_profile", "target_tokens", "actual_tokens", + "tokenizer", "builder_runtime_build", "builder_runtime_build_sha256", + "builder_install_trust", + "builder_install_trust_sha256", + }, + ) + preflight.validate_prompt_provenance( + provenance_path, + prompt=output, + corpus_name="correctness-prose.txt", + corpus_sha256=trace.CORPUS_SHA256["correctness-prose.txt"], + model_sha256=trace.MODEL_SHA256, + context=3, + decode_steps=1, + target_tokens=2, + builder_approval_id=TEST_PROMPT_BUILDER_POLICY_ID, + builder_policy=builder_policy, + builder_policy_sha256=builder_policy_sha256, + path_resolver=lambda path, _label: path.resolve(), + ) + + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "descriptor identity test") + def test_model_descriptor_binds_bytes_and_rejects_path_replacement(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + model = root / "model.gguf" + replacement = root / "replacement.gguf" + model.write_bytes(b"approved model bytes") + replacement.write_bytes(b"replacement bytes") + descriptor, identity = run_llama.open_model_descriptor(model) + try: + self.assertEqual( + run_llama.sha256_descriptor(descriptor), + hashlib.sha256(b"approved model bytes").hexdigest(), + ) + os.replace(replacement, model) + self.assertEqual( + run_llama.sha256_descriptor(descriptor), + hashlib.sha256(b"approved model bytes").hexdigest(), + ) + if sys.platform == "linux" and os.environ.get("DSV41_NATIVE_TRACE_BINARY"): + environment = dict(os.environ) + environment["DSV41_MODEL_DESCRIPTOR"] = str(descriptor) + environment["DSV41_MODEL_DESCRIPTOR_IDENTITY"] = trace.canonical_json(identity) + native = subprocess.run( + [ + str(Path(os.environ["DSV41_NATIVE_TRACE_BINARY"]).resolve(strict=True)), + "--dsv41-test-model-descriptor", + str(model), + ], + check=False, + capture_output=True, + text=True, + env=environment, + pass_fds=(descriptor,), + ) + self.assertNotEqual(native.returncode, 0) + self.assertIn("held model descriptor policy is invalid", native.stderr) + with self.assertRaisesRegex( + run_llama.PreflightError, "linked pathname|pathname identity changed"): + run_llama.verify_model_descriptor(descriptor, identity) + finally: + os.close(descriptor) + + @unittest.skipUnless(sys.platform.startswith(("darwin", "linux")), "descriptor identity test") + def test_model_descriptor_rejects_symlink_and_in_place_mutation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + model = root / "model.gguf" + alias = root / "alias.gguf" + model.write_bytes(b"approved model bytes") + alias.symlink_to(model) + with self.assertRaisesRegex( + run_llama.PreflightError, "symbolic-link aliases"): + run_llama.open_model_descriptor(alias) + descriptor, identity = run_llama.open_model_descriptor(model) + try: + model.write_bytes(b"mutated model bytes") + with self.assertRaisesRegex( + run_llama.PreflightError, "identity or bytes changed"): + run_llama.verify_model_descriptor(descriptor, identity) + finally: + os.close(descriptor) + + @unittest.skipUnless(sys.platform == "linux", "watchdog pidfd authority test") + def test_watchdog_namespace_authority_rejects_reuse_and_executable_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + procfs = root / "proc" + watchdog_pid = 123 + guardian_pid = 456 + child_pid = 457 + descriptor = os.open("/dev/null", os.O_RDONLY) + + def proc_stat(pid: int, parent: int, group: int, start: int) -> str: + fields = ["S", str(parent), str(group), *(["0"] * 16), str(start)] + return f"{pid} (test) " + " ".join(fields) + "\n" + + try: + for pid in (watchdog_pid, guardian_pid, child_pid): + (procfs / str(pid)).mkdir(parents=True) + (procfs / "self" / "fdinfo").mkdir(parents=True) + (procfs / str(watchdog_pid) / "stat").write_text( + proc_stat(watchdog_pid, 1, watchdog_pid, 1000), encoding="ascii") + (procfs / str(guardian_pid) / "stat").write_text( + proc_stat(guardian_pid, watchdog_pid, guardian_pid, 2000), encoding="ascii") + (procfs / str(child_pid) / "stat").write_text( + proc_stat(child_pid, guardian_pid, guardian_pid, 3000), encoding="ascii") + executable = root / "python3" + executable.write_bytes(b"python") + (procfs / str(watchdog_pid) / "exe").symlink_to(executable) + command = b"python3\0watchdog.py\0" + (procfs / str(watchdog_pid) / "cmdline").write_bytes(command) + def fake_pidfd_open(_pid: int, _flags: int) -> int: + retained = os.dup(descriptor) + (procfs / "self" / "fdinfo" / str(retained)).write_text( + f"Pid:\t{watchdog_pid}\n", encoding="ascii") + return retained + + audit = copy.deepcopy(AUDIT_RECORDS["watchdog"]["data"]) + audit.update({ + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": 1000, + "watchdog_executable_path": str(executable), + "watchdog_command_sha256": preflight.sha256_bytes(command), + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": guardian_pid, + }) + retained, authority = preflight.open_watchdog_namespace_authority( + audit, + procfs_root=procfs, + pidfd_open=fake_pidfd_open, + ) + try: + self.assertEqual(authority["watchdog_pid"], watchdog_pid) + preflight.verify_watchdog_namespace_authority( + retained, authority, procfs_root=procfs) + finally: + os.close(retained) + + reused = copy.deepcopy(audit) + reused["watchdog_start_time_ticks"] = 999 + with self.assertRaisesRegex(preflight.PreflightError, "start time"): + preflight.open_watchdog_namespace_authority( + reused, + procfs_root=procfs, + pidfd_open=fake_pidfd_open, + ) + other_executable = root / "other-python" + other_executable.write_bytes(b"other") + mismatched = copy.deepcopy(audit) + mismatched["watchdog_executable_path"] = str(other_executable) + with self.assertRaisesRegex(preflight.PreflightError, "executable"): + preflight.open_watchdog_namespace_authority( + mismatched, + procfs_root=procfs, + pidfd_open=fake_pidfd_open, + ) + finally: + os.close(descriptor) + + @unittest.skipUnless( + sys.platform == "linux" and + os.environ.get("DSV41_NATIVE_CONTAINMENT_HELPER") and + os.environ.get("DSV41_NATIVE_TRACE_BINARY"), + "native Linux watchdog namespace validation was not executed on this host", + ) + def test_native_watchdog_validation_crosses_private_pid_namespace(self) -> None: + import fcntl + + helper = Path(os.environ["DSV41_NATIVE_CONTAINMENT_HELPER"]).resolve(strict=True) + binary = Path(os.environ["DSV41_NATIVE_TRACE_BINARY"]).resolve(strict=True) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + lease = root / "watchdog.lease" + heartbeat = root / "watchdog.heartbeat" + audit = root / "watchdog.jsonl" + data_path = root / "watchdog-data.json" + lease.write_text("{}\n", encoding="ascii") + audit_line = '{"event":"preflight","timestamp":"1970-01-01T00:00:01.000Z"}\n' + audit.write_text(audit_line, encoding="ascii") + audit.chmod(0o600) + audit_descriptor = os.open(audit, os.O_RDONLY) + pidfd = os.pidfd_open(os.getpid()) + helper_descriptor = os.open(helper, os.O_RDONLY) + try: + fcntl.flock(audit_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + audit_status = audit.stat() + heartbeat.write_text( + json.dumps({ + "format": "strix-memory-watchdog-heartbeat", + "version": 2, + "lease_id": "1" * 32, + "sequence": 1, + "state": "active", + "updated_at": "1970-01-01T00:00:01.000Z", + "updated_monotonic_ns": time.monotonic_ns(), + "watchdog_pid": os.getpid(), + "watchdog_start_time_ticks": 1000, + "child_pid": os.getpid() + 2, + "child_process_group_id": os.getpid() + 1, + "sample": { + "audit_record_sha256": trace.sha256_bytes( + audit_line.encode("ascii")), + }, + }, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + command = ["python3", "run_matrix.py"] + data = copy.deepcopy(AUDIT_RECORDS["watchdog"]["data"]) + data.update({ + "lease_id": "1" * 32, + "lease_path": str(lease), + "watchdog_pid": os.getpid(), + "watchdog_start_time_ticks": 1000, + "watchdog_command_sha256": "7" * 64, + "watchdog_executable_path": str(Path(sys.executable).resolve()), + "watchdog_script_path": str( + (Path(__file__).parents[1] / "scripts" / "strix_memory_watchdog.py").resolve()), + "guardian_pid": os.getpid() + 1, + "child_pid": os.getpid() + 2, + "child_process_group_id": os.getpid() + 1, + "command": command, + "child_command_sha256": trace.sha256_bytes( + json.dumps(command, ensure_ascii=True, separators=(",", ":")).encode("ascii")), + "heartbeat_path": str(heartbeat), + "max_heartbeat_age_seconds": 5.0, + "audit_live_path": str(audit), + "audit_device": audit_status.st_dev, + "audit_inode": audit_status.st_ino, + "audit_uid": audit_status.st_uid, + "audit_mode": 0o600, + "audit_fd": audit_descriptor, + "namespace_authority": { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": pidfd, + "host_procfs_root": "/proc", + "watchdog_pid": os.getpid(), + "watchdog_process_group_id": os.getpgrp(), + "watchdog_start_time_ticks": 1000, + "watchdog_executable_path": str(Path(sys.executable).resolve()), + "watchdog_command_sha256": "7" * 64, + "guardian_pid": os.getpid() + 1, + "child_pid": os.getpid() + 2, + "child_process_group_id": os.getpid() + 1, + }, + }) + data_path.write_text( + json.dumps(data, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + environment = dict(os.environ) + environment.update({ + "DSV41_WATCHDOG_PIDFD": str(pidfd), + "STRIX_MEMORY_WATCHDOG_LEASE_PATH": str(lease), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH": str(heartbeat), + "STRIX_MEMORY_WATCHDOG_AUDIT_PATH": str(audit), + "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS": "5.0", + }) + contained = trace._run_contained_process( + [str(binary), "--dsv41-test-watchdog", str(data_path)], + label="native watchdog namespace test", + timeout=20, + input_data=None, + launch={ + "executable": str(binary), + "pass_fds": (pidfd,), + "_containment_helper_path": str(helper), + "_containment_helper_descriptor": helper_descriptor, + "env": environment, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + }, + ) + try: + if contained.primary_error is not None: + raise contained.primary_error + self.assertEqual(contained.integrity_failures, []) + self.assertIsNotNone(contained.result) + assert contained.result is not None + self.assertEqual( + contained.result.returncode, + 0, + contained.result.stderr.decode("utf-8", "replace"), + ) + binding = trace.strict_json_loads(contained.result.stdout.decode("ascii")) + self.assertTrue(binding["private_procfs"]) + self.assertEqual(binding["local_pid"], 2) + self.assertEqual(binding["local_parent_pid"], 1) + self.assertEqual(binding["local_pid"], binding["local_process_group_id"]) + self.assertEqual(binding["namespace_pids"][-1], binding["local_pid"]) + finally: + failures = trace._close_process_containment( + contained.containment, + quiescence_proven=contained.quiescence_proven, + ) + self.assertEqual(failures, []) + finally: + os.close(helper_descriptor) + os.close(pidfd) + fcntl.flock(audit_descriptor, fcntl.LOCK_UN) + os.close(audit_descriptor) + + def test_rejects_cross_runtime_and_unknown_audit_envelopes(self) -> None: + for mutation, message in ( + (lambda value: value["audits"].update({"unknown": {}}), "audit envelope"), + (lambda value: value["audits"]["pre"].update({"watchdog": {}}), "audit reference")): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + mutation(trace_manifest) + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_requires_no_external_cache_or_state_storage(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest("ds4") + trace_manifest["storage_policy"]["external_cache_paths"] = ["/Users/oracle/cache"] + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "storage policy"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["memory"])) + del record["storage_policy"] + replace_audit_record(root, "pre", "memory", record) + with self.assertRaisesRegex(trace.TraceError, "storage_policy"): + trace.TraceBundle(root) + + def test_accepts_authentic_watchdog_lease_fields(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + trace.TraceBundle(root) + + for field, value, message in ( + ("file_inode", True, "file_inode is invalid"), + ("file_mode", 0o644, "file mode is invalid"), + ("audit_sha256", "f" * 64, "live and embedded audit SHA-256 differ")): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = copy.deepcopy(AUDIT_RECORDS["watchdog"]) + record["data"]["audit"]["path"] = ( + f"audits/{phase}/{WATCHDOG_JSONL_SHA256}.jsonl") + record["data"][field] = value + replace_audit_record(root, phase, "watchdog", record) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_enforces_watchdog_final_error_classification(self) -> None: + self.assertEqual(trace.WATCHDOG_REQUIRED_ERROR_CLASSIFICATIONS, { + "configuration_error", + "internal_error", + "launch_error", + "lease_error", + "termination_timeout", + }) + self.assertEqual(trace.WATCHDOG_OPTIONAL_ERROR_CLASSIFICATIONS, { + "procfs_error", + "signal_error", + }) + terminal = { + key: copy.deepcopy(value) + for key, value in WATCHDOG_EVENTS[1].items() + if key not in {"command", "event_id", "parent_event_sha256", "record_sha256"} + } + terminal.update({ + "event": "final", + "classification": "internal_error", + "exit_code": 1, + "error": "test internal error", + }) + + def assert_watchdog_final_rejected(candidate: dict[str, object], message: str) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + with self.assertRaisesRegex(trace.TraceError, message): + trace.command_validate(Namespace(bundle=root)) + + def assert_watchdog_final_valid(candidate: dict[str, object]) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + replace_watchdog_events(root, phase, [*WATCHDOG_EVENTS, candidate]) + trace.TraceBundle(root) + with mock.patch("sys.stdout", new_callable=io.StringIO): + self.assertEqual(trace.command_validate(Namespace(bundle=root)), 0) + + assert_watchdog_final_valid(terminal) + + for classification, error in ( + ("procfs_error", None), + ("procfs_error", "initial snapshot failed"), + ("signal_error", None), + ("signal_error", "cannot signal process group")): + with self.subTest(classification=classification, error=error): + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + assert_watchdog_final_valid(candidate) + + candidate = copy.deepcopy(terminal) + candidate.update({ + "classification": "signal_error", + "error": "primary signal failure", + "secondary_errors": [{ + "component": "audit", + "detail": "secondary audit failure", + }], + }) + assert_watchdog_final_valid(candidate) + + for classification, error in ( + ("internal_error", "internal failure"), + ("signal_error", None)): + with self.subTest(classification=classification, secondary=True): + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + candidate["secondary_errors"] = [{ + "component": "audit", + "detail": "secondary audit failure", + }] + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + assert_watchdog_final_rejected(candidate, "require a primary signal error") + + for classification, error, secondary_errors, message in ( + ("child_exit", None, None, "require a primary signal error"), + ("internal_error", "internal failure", None, "require a primary signal error"), + ("signal_error", None, None, "require a primary signal error"), + ("signal_error", "primary signal failure", None, "secondary errors are invalid"), + ("signal_error", "primary signal failure", [], "secondary errors are invalid")): + with self.subTest( + classification=classification, + secondary_errors=secondary_errors): + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + candidate["secondary_errors"] = secondary_errors + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + assert_watchdog_final_rejected(candidate, message) + + for classification, error in ( + ("internal_error", None), + ("child_exit", "fabricated error")): + with self.subTest(classification=classification): + candidate = copy.deepcopy(terminal) + candidate["classification"] = classification + if error is None: + candidate.pop("error") + else: + candidate["error"] = error + assert_watchdog_final_rejected(candidate, "error presence does not match") + + def test_rejects_boolean_accelerator_identities(self) -> None: + for runtime, field in ( + ("llama.cpp", "gpu_id"), + ("ds4", "metal_registry_id"), + ("ds4", "recommended_max_working_set_bytes")): + with self.subTest(runtime=runtime, field=field), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + trace_manifest = manifest(runtime) + trace_manifest["accelerator"][field] = True + with trace.TraceBundleWriter(root, trace_manifest) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "invalid"): + trace.TraceBundle(root) + + def test_rejects_unknown_watchdog_event_fields(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + events = json.loads(json.dumps(WATCHDOG_EVENTS)) + events[0]["unknown"] = True + data = "".join( + json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + for event in events + ).encode("ascii") + digest = trace.sha256_bytes(data) + for phase in ("pre", "post"): + jsonl = root / "audits" / phase / f"{digest}.jsonl" + jsonl.write_bytes(data) + record = json.loads(json.dumps(AUDIT_RECORDS["watchdog"])) + record["data"]["audit_sha256"] = digest + record["data"]["audit"]["path"] = f"audits/{phase}/{digest}.jsonl" + record["data"]["audit"]["sha256"] = digest + record["data"]["audit"]["event_count"] = len(events) + replace_audit_record(root, phase, "watchdog", record) + with self.assertRaisesRegex(trace.TraceError, "unexpected unknown"): + trace.TraceBundle(root) + + def test_ds4_runner_path_must_match_attested_storage(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + for phase in ("pre", "post"): + record = json.loads(json.dumps(DS4_AUDIT_RECORDS["runner"])) + record["data"]["runner_script"] = "/Users/oracle/other/run_ds4.py" + replace_audit_record(root, phase, "runner", record) + with self.assertRaisesRegex(trace.TraceError, "runner_script mismatch"): + trace.TraceBundle(root) + + def test_unapproved_ds4_exporter_is_not_executed(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + exporter = root / "exporter" + exporter.write_text("#!/bin/sh\nexit 0\n", encoding="ascii") + exporter.chmod(0o755) + argv = [ + "run_ds4.py", + "--repo", str(root), + "--model", str(root / "model.gguf"), + "--prompt", str(root / "prompt.txt"), + "--output", str(root / "output"), + "--exporter", str(exporter), + "--exporter-sha256", trace.sha256_file(exporter), + "--corpus-name", "correctness-prose.txt", + "--corpus-sha256", trace.CORPUS_SHA256["correctness-prose.txt"], + "--prompt-provenance", str(root / "prompt.json"), + "--ds4-exporter-policy-id", TEST_DS4_EXPORTER_POLICY_ID, + "--prompt-builder-policy-id", TEST_PROMPT_BUILDER_POLICY_ID, + "--approval-policy", str(root / "approval.json"), + "--approval-signature", str(root / "approval.sig"), + "--approval-principal", "unapproved", + "--signer-principal", self.signer_principals["ds4"], + "--signing-key", str(self.signing_keys["ds4"]), + "--execution-challenge", TEST_CHALLENGE, + "--run-id", TEST_RUN_IDS["ds4"], + "--authorization-issued-unix", str(TEST_AUTH_ISSUED), + "--authorization-expires-unix", str(TEST_AUTH_EXPIRES), + "--preflight-only", + ] + with mock.patch.object(sys, "argv", argv), mock.patch.object( + sys, "stderr", io.StringIO()), mock.patch.object( + run_ds4, "query_accelerator_attestation") as query: + self.assertEqual(run_ds4.main(), 1) + query.assert_not_called() + + def test_rejects_wrong_component_schema_and_same_bundle_compare(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("llama.cpp")) as writer: + add_required_events(writer) + events_path = root / trace.EVENTS_NAME + events = [ + json.loads(line) + for line in events_path.read_text(encoding="ascii").splitlines() + ] + expert = next(event for event in events if event["component"] == "expert.ids") + expert["dtype"] = "f32" + events_path.write_text( + "".join(trace.canonical_json(event) + "\n" for event in events), + encoding="ascii", + ) + with self.assertRaisesRegex(trace.TraceError, "expert.ids dtype"): + trace.TraceBundle(root) + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + bundle = trace.TraceBundle(root) + result = trace.report(bundle, bundle) + self.assertEqual(result["first_divergence"]["classification"], "artifact_identity") + + def test_rejects_empty_variable_width_components(self) -> None: + for component in ("attn.source", "attn.candidate_blocks", "attn.candidates"): + with self.subTest(component=component), tempfile.TemporaryDirectory() as temp: + writer = trace.TraceBundleWriter(Path(temp) / "trace", manifest()) + with self.assertRaisesRegex(trace.TraceError, "nonzero"): + writer.add_event( + component=component, + phase="prefill", + step=0, + token_start=0, + token_count=2, + layer=20, + dtype="i32", + shape=[0, 2], + data=b"", + ) + writer.events.close() + + def test_rejects_malformed_raw_attention_sources(self) -> None: + cases = ( + ( + "width", + 0, + [trace.RAW_ATTENTION_WIDTH - 1, 2], + [trace.RAW_ATTENTION_WIDTH + 2] * ((trace.RAW_ATTENTION_WIDTH - 1) * 2), + "raw attn.source shape", + ), + ( + "future ubatch row", + 0, + [trace.RAW_ATTENTION_WIDTH, 2], + [trace.RAW_ATTENTION_WIDTH + 1] + + [trace.RAW_ATTENTION_WIDTH + 2] * (trace.RAW_ATTENTION_WIDTH * 2 - 1), + "invalid row", + ), + ( + "layer mismatch", + 1, + [trace.RAW_ATTENTION_WIDTH, 2], + [0] + [trace.RAW_ATTENTION_WIDTH + 2] * (trace.RAW_ATTENTION_WIDTH * 2 - 1), + "differs between layers 0 and 1", + ), + ) + for name, layer, shape, values, message in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + replace_event_blob( + root, + component="attn.source", + phase="prefill", + layer=layer, + shape=shape, + data=struct.pack("<" + "i" * len(values), *values), + ) + with self.assertRaisesRegex(trace.TraceError, message): + trace.TraceBundle(root) + + def test_compares_raw_attention_sources_in_every_prefill_chunk(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest()) as writer: + add_required_events(writer) + events = [ + json.loads(line) + for line in (root / trace.EVENTS_NAME).read_text(encoding="ascii").splitlines() + ] + split_events = [] + for event in events: + if event["component"] != "attn.source" or event["phase"] != "prefill" or ( + event["layer"] not in trace.RAW_ATTENTION_LAYERS): + split_events.append(event) + continue + for token_start in (0, 1): + values = [0] * trace.RAW_ATTENTION_WIDTH + if event["layer"] == 0 and token_start == 0: + values[0] = 1 + data = struct.pack("<" + "i" * len(values), *values) + digest = trace.sha256_bytes(data) + (root / trace.BLOBS_DIR / f"{digest}.bin").write_bytes(data) + split = dict(event) + split.update({ + "token_start": token_start, + "token_count": 1, + "shape": [trace.RAW_ATTENTION_WIDTH, 1], + "byte_count": len(data), + "sha256": digest, + "blob": f"{trace.BLOBS_DIR}/{digest}.bin", + }) + split_events.append(split) + (root / trace.EVENTS_NAME).write_text( + "".join(json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + for event in split_events), + encoding="ascii", + ) + manifest_record = json.loads((root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + manifest_record["event_count"] = len(split_events) + (root / trace.MANIFEST_NAME).write_text( + json.dumps(manifest_record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + with self.assertRaisesRegex(trace.TraceError, "differs between layers 0 and 1"): + trace.TraceBundle(root) + + def test_rejects_zero_dimensions_globally(self) -> None: + with tempfile.TemporaryDirectory() as temp: + writer = trace.TraceBundleWriter(Path(temp) / "trace", manifest()) + with self.assertRaisesRegex(trace.TraceError, "nonzero"): + writer.add_event( + component="prompt.bytes", + phase="input", + step=0, + token_start=0, + token_count=1, + layer=None, + dtype="bytes", + shape=[0], + data=b"", + ) + writer.events.close() + + def test_rejects_dirty_ds4_checkout(self) -> None: + original = run_ds4.git_output + try: + run_ds4.git_output = lambda checkout, *args: ( + trace.DS4_REVISION if args == ("rev-parse", "HEAD") else " M runtime.py") + with self.assertRaisesRegex(preflight.PreflightError, "tracked or untracked"): + run_ds4.verify_checkout(Path("/tmp/ds4")) + finally: + run_ds4.git_output = original + + def test_verifies_pinned_ds4_anchor_hashes(self) -> None: + with tempfile.TemporaryDirectory() as temp: + checkout = Path(temp) + anchor = checkout / "tests" / "fixture.vec" + anchor.parent.mkdir(parents=True) + anchor.write_bytes(b"fixture") + expected = trace.sha256_bytes(b"fixture") + original = verify_ds4_anchors.git_output + try: + verify_ds4_anchors.git_output = lambda checkout, *args: ( + trace.DS4_REVISION if args == ("rev-parse", "HEAD") else "") + result = verify_ds4_anchors.verify( + checkout, + anchors={"tests/fixture.vec": expected}, + ) + self.assertEqual(result["status"], "ANCHORS VERIFIED") + anchor.write_bytes(b"changed") + with self.assertRaisesRegex(verify_ds4_anchors.AnchorError, "SHA-256 mismatch"): + verify_ds4_anchors.verify( + checkout, + anchors={"tests/fixture.vec": expected}, + ) + finally: + verify_ds4_anchors.git_output = original + + def test_embeds_rewritten_watchdog_audit_content(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "source" + source.mkdir() + audits = {} + for kind in ("memory", "swap", "watchdog"): + record = json.loads(json.dumps(AUDIT_RECORDS[kind])) + if kind == "watchdog": + jsonl = source / "watchdog-events.jsonl" + jsonl.write_bytes(WATCHDOG_JSONL) + record["data"].pop("audit") + record["data"]["audit_path"] = str(jsonl) + record["data"]["audit_sha256"] = WATCHDOG_JSONL_SHA256 + record["data"]["audit_event_count"] = len(WATCHDOG_EVENTS) + path = source / f"{kind}.json" + path.write_text( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + audits[kind] = str(path) + references = preflight.embed_audits(root / "trace", "pre", audits) + embedded = json.loads( + (root / "trace" / references["watchdog"]["path"]).read_text(encoding="ascii")) + self.assertIn("audit", embedded["data"]) + self.assertNotIn("audit_path", embedded["data"]) + self.assertEqual( + trace.sha256_bytes( + (root / "trace" / references["watchdog"]["path"]).read_bytes()), + references["watchdog"]["sha256"], + ) + + def test_rejects_unapproved_ds4_exporter(self) -> None: + with self.assertRaisesRegex(trace.TraceError, "not trusted"): + trace.ds4_exporter_approval("missing", policies={}) + + def test_bundle_validation_and_comparison_reject_unapproved_ds4_exporter(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + ds4_root = root / "ds4" + with trace.TraceBundleWriter(ds4_root, manifest("ds4")) as writer: + add_required_events(writer) + self._seal_test_bundle(ds4_root) + verifier = self._verifier_for_runtime("ds4") + verifier = replace(verifier, ds4_exporter_policies={}) + with self.assertRaisesRegex(trace.TraceError, "not trusted"): + self._trace_bundle_class(ds4_root, verifier=verifier) + + def test_ds4_oracle_trust_bindings_reject_mutation_before_signing(self) -> None: + mutations = ( + ( + lambda value: value["oracle"].update({"exporter_approval_id": "other"}), + "approval differs", + ), + ( + lambda value: value["oracle"].update({"exporter_approval_sha256": "f" * 64}), + "approval differs", + ), + ( + lambda value: value["oracle"].update({"install_trust_sha256": "f" * 64}), + "install trust", + ), + ( + lambda value: value["oracle"].update({"runtime_build_sha256": "f" * 64}), + "runtime build", + ), + ( + lambda value: value["oracle"].update({"runtime_receipt_sha256": "f" * 64}), + "runtime evidence", + ), + ( + lambda value: value["authorization"]["approvals"]["ds4_exporter"].update({ + "install_trust_sha256": "f" * 64, + }), + "install trust", + ), + ) + for mutate, message in mutations: + with self.subTest(message=message), tempfile.TemporaryDirectory() as temp: + record = manifest("ds4") + mutate(record) + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, record) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, message): + self._seal_test_bundle(root) + + def test_ds4_runner_trust_binding_rejects_mutation_before_signing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + with trace.TraceBundleWriter(root, manifest("ds4")) as writer: + add_required_events(writer) + manifest_record = trace.strict_json_loads( + (root / trace.MANIFEST_NAME).read_text(encoding="ascii")) + original_path = root / manifest_record["audits"]["pre"]["runner"]["path"] + runner = json.loads(json.dumps(DS4_AUDIT_RECORDS["runner"])) + runner["data"]["exporter_install_trust_sha256"] = "f" * 64 + replace_audit_record(root, "pre", "runner", runner) + original_path.unlink() + with self.assertRaisesRegex(trace.TraceError, "install.trust"): + self._seal_test_bundle(root) + + def test_rejects_preflight_audit_mutation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + audits = {} + for kind in ("memory", "swap", "watchdog"): + record = json.loads(json.dumps(AUDIT_RECORDS[kind])) + if kind == "watchdog": + jsonl = root / "watchdog-events.jsonl" + jsonl.write_bytes(WATCHDOG_JSONL) + record["data"]["audit_path"] = str(jsonl) + record["data"]["audit_sha256"] = WATCHDOG_JSONL_SHA256 + record["data"]["audit_event_count"] = len(WATCHDOG_EVENTS) + path = root / f"{kind}.json" + path.write_text( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + audits[kind] = str(path) + digests = preflight.seal_audits(audits) + memory = Path(audits["memory"]) + memory.chmod(0o644) + memory.write_text("{}\n", encoding="ascii") + with self.assertRaisesRegex(preflight.PreflightError, "changed during runtime"): + preflight.verify_sealed_audits(audits, digests) + + def test_rejects_symlinked_bundle_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + bundle = root / "bundle" + with trace.TraceBundleWriter(bundle, manifest()) as writer: + add_required_events(writer) + provenance = next((bundle / "provenance").iterdir()) + outside = root / "outside.json" + outside.write_bytes(provenance.read_bytes()) + provenance.unlink() + provenance.symlink_to(outside) + with self.assertRaisesRegex(trace.TraceError, "must not use symlinks"): + trace.TraceBundle(bundle) + + def test_rejects_weakened_coverage_contract(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + weakened = manifest() + del weakened["expected"]["components"]["expert.ids"]["decode"] + with trace.TraceBundleWriter(root, weakened) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "coverage contract"): + trace.TraceBundle(root) + + def test_rejects_prompt_token_count_not_bound_to_provenance(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + mismatched = manifest() + mismatched["expected"]["prompt_tokens"] = 1 + with trace.TraceBundleWriter(root, mismatched) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "prompt provenance"): + trace.TraceBundle(root) + + def test_watchdog_stat_parser_handles_parentheses(self) -> None: + fields = ["S", *[str(value) for value in range(4, 23)]] + self.assertEqual(preflight.proc_start_time_ticks(f"123 (watch) dog) {' '.join(fields)}"), 22) + + def test_report_generation_passes_identical_bundles(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + with trace.TraceBundleWriter(left, manifest("ds4")) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, manifest("llama.cpp")) as writer: + add_required_events(writer) + result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) + self.assertEqual(result["status"], "TARGET PASS") + self.assertEqual(result["events_compared"], 259) + self.assertIsNone(result["first_divergence"]) + + def test_local_bringup_reports_do_not_claim_cross_runtime_pass(self) -> None: + with tempfile.TemporaryDirectory() as temp: + first = Path(temp) / "first" + second = Path(temp) / "second" + with trace.TraceBundleWriter(first, manifest("llama.cpp")) as writer: + add_required_events(writer) + second_manifest = manifest("llama.cpp") + second_manifest["authorization"]["run_id"] = "strix-llama-test-run-2" + with trace.TraceBundleWriter(second, second_manifest) as writer: + add_required_events(writer) + result = trace.local_report( + trace.TraceBundle(first), + trace.TraceBundle(second), + "self-consistency", + ) + self.assertEqual(result["status"], "BRINGUP PASS") + self.assertNotEqual(result["status"], "TARGET PASS") + self.assertEqual(result["cross_runtime_status"], "INCOMPLETE") + with mock.patch("sys.stdout", new_callable=io.StringIO): + self.assertEqual( + trace.command_compare_local(Namespace( + mode="self-consistency", + left=first, + right=second, + left_signer_principal=self.signer_principal, + right_signer_principal=self.signer_principal, + execution_challenge=TEST_CHALLENGE, + left_run_id=TEST_RUN_IDS["llama.cpp"], + right_run_id="strix-llama-test-run-2", + report=None, + )), + 0, + ) + + def test_local_base_regression_requires_attested_oracle_revision(self) -> None: + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / "base" + integrated = Path(temp) / "integrated" + base_manifest = manifest("llama.cpp") + base_manifest["revision"] = "b" * 40 + base_manifest["candidate"]["revision"] = "b" * 40 + for library in base_manifest["build"]["runtime_libraries"]: + if library["component"] in {"llama-common", "ggml-base"}: + library["revision"] = "b" * 40 + base_manifest["build"]["runtime_libraries_post"] = copy.deepcopy( + base_manifest["build"]["runtime_libraries"]) + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": "b" * 40, + "profile": base_manifest["build"]["runtime_profile"]["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in base_manifest["build"]["runtime_libraries"] + ], + key=lambda item: item["component"], + ), + } + receipt_sha256 = trace.sha256_bytes(trace.canonical_json(receipt).encode("ascii")) + base_manifest["build"]["runtime_receipt_sha256"] = receipt_sha256 + base_manifest["candidate"]["runtime_libraries_sha256"] = trace.sha256_bytes( + trace.canonical_json({ + "pre": base_manifest["build"]["runtime_libraries"], + "post": base_manifest["build"]["runtime_libraries_post"], + }).encode("ascii")) + base_manifest["candidate"]["runtime_receipt_sha256"] = receipt_sha256 + base_verifier = self._verifier_for_runtime( + "llama.cpp", manifest_record=base_manifest) + _candidate_policy, candidate_policy_sha256 = trace.candidate_exporter_approval( + TEST_CANDIDATE_EXPORTER_POLICY_ID, + policies=base_verifier.candidate_exporter_policies, + ) + base_manifest["candidate"]["exporter_approval_sha256"] = candidate_policy_sha256 + base_manifest["authorization"]["approvals"]["candidate_exporter"] = ( + trace.approval_binding( + "candidate_exporter", + TEST_CANDIDATE_EXPORTER_POLICY_ID, + candidate_policy_sha256, + base_manifest["candidate"]["install_trust_sha256"], + ) + ) + with trace.TraceBundleWriter(base, base_manifest) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(integrated, manifest("llama.cpp")) as writer: + add_required_events(writer) + result = trace.local_report( + trace.TraceBundle(base), + trace.TraceBundle(integrated), + "base-regression", + ) + self.assertEqual(result["status"], "BRINGUP PASS") + self.assertEqual(result["cross_runtime_status"], "INCOMPLETE") + + def test_manifest_mismatch_is_classified(self) -> None: + with tempfile.TemporaryDirectory() as temp: + left = Path(temp) / "left" + right = Path(temp) / "right" + left_manifest = manifest("ds4") + right_prompt = b"abd" + right_manifest = manifest("llama.cpp", right_prompt) + with trace.TraceBundleWriter(left, left_manifest) as writer: + add_required_events(writer) + with trace.TraceBundleWriter(right, right_manifest) as writer: + add_required_events(writer, prompt=right_prompt) + result = trace.report(trace.TraceBundle(left), trace.TraceBundle(right)) + self.assertEqual(result["first_divergence"]["classification"], "prompt_identity") + + def test_identically_incomplete_decode_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "trace" + incomplete = manifest(context=4, decode_steps=2) + with trace.TraceBundleWriter(root, incomplete) as writer: + add_required_events(writer) + with self.assertRaisesRegex(trace.TraceError, "decode step coverage"): + trace.TraceBundle(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_strix_memory_watchdog.py b/tests/test_strix_memory_watchdog.py new file mode 100644 index 000000000000..f9cba175a8d1 --- /dev/null +++ b/tests/test_strix_memory_watchdog.py @@ -0,0 +1,2415 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import fcntl +import hashlib +import io +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[1] + / "scripts" + / "strix_memory_watchdog.py" +) +SPEC = importlib.util.spec_from_file_location( + "strix_memory_watchdog", SCRIPT_PATH +) +assert SPEC is not None +assert SPEC.loader is not None +watchdog = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = watchdog +SPEC.loader.exec_module(watchdog) + + +def snapshot( + used_bytes: int, + *, + total_bytes: int = 200, + active_swaps: tuple[str, ...] = (), +) -> Any: + return watchdog.HostSnapshot( + total_bytes=total_bytes, + available_bytes=total_bytes - used_bytes, + active_swaps=active_swaps, + ) + + +class SequenceReader: + def __init__(self, values: list[Any]): + self.values = values + self.index = 0 + + def read_snapshot(self) -> Any: + index = min(self.index, len(self.values) - 1) + self.index += 1 + value = self.values[index] + if isinstance(value, Exception): + raise value + return value + + +class FakeClock: + def __init__(self): + self.value = 0.0 + + def monotonic(self) -> float: + return self.value + + def sleep(self, seconds: float) -> None: + self.value += seconds + + +class FakeProcess: + def __init__(self, returncode: int | None = None): + self.pid = 4321 + self.returncode = returncode + + def poll(self) -> int | None: + return self.returncode + + def wait(self, timeout: float | None = None) -> int: + if self.returncode is None: + raise subprocess.TimeoutExpired("fake", timeout or 0.0) + return self.returncode + + +class Harness: + def __init__( + self, + values: list[Any], + process: FakeProcess, + signal_handler: Any | None = None, + ): + self.reader = SequenceReader(values) + self.process = process + self.signal_handler = signal_handler + self.clock = FakeClock() + self.stream = io.StringIO() + self.launched = False + self.signals: list[int] = [] + fixed_time = datetime(2026, 1, 1, tzinfo=timezone.utc) + self.audit = watchdog.AuditLogger( + self.stream, wall_clock=lambda: fixed_time + ) + + def launcher(self, command: tuple[str, ...], **kwargs: Any) -> FakeProcess: + self.launched = True + self.command = command + self.launch_kwargs = kwargs + return self.process + + def signal_group(self, process_group_id: int, signal_number: int) -> str: + self.signals.append(signal_number) + if self.signal_handler is not None: + self.signal_handler(self.process, signal_number) + return f"{signal.Signals(signal_number).name.lower()}_sent" + + def group_alive(self, process_group_id: int) -> bool: + return self.process.returncode is None + + def run(self, **overrides: Any) -> int: + config = watchdog.WatchdogConfig( + command=("fake-command",), + soft_bytes=100, + emergency_bytes=150, + grace_seconds=2, + sample_interval_seconds=1, + **overrides, + ) + return watchdog.run_watchdog( + config, + reader=self.reader, + audit=self.audit, + launcher=self.launcher, + signal_group=self.signal_group, + group_alive=self.group_alive, + monotonic=self.clock.monotonic, + sleeper=self.clock.sleep, + ) + + def records(self) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in self.stream.getvalue().splitlines() + ] + + +class TestProcfsParsing(unittest.TestCase): + def test_parses_meminfo_as_integer_bytes_and_allows_zero_swap(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "meminfo").write_text( + "MemTotal: 131072 kB\n" + "MemFree: 4096 kB\n" + "MemAvailable: 32768 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + + result = watchdog.ProcfsReader(root).read_snapshot() + + self.assertEqual(result.total_bytes, 131072 * 1024) + self.assertEqual(result.available_bytes, 32768 * 1024) + self.assertEqual(result.used_bytes, 98304 * 1024) + self.assertEqual(result.active_swaps, ()) + + def test_rejects_active_swap_entry(self) -> None: + content = ( + "Filename Type Size Used Priority\n" + "/swapfile file 1048572 0 -2\n" + ) + self.assertEqual( + watchdog.ProcfsReader._parse_swaps(content), + ("/swapfile",), + ) + + def test_rejects_malformed_or_missing_procfs_data(self) -> None: + with self.assertRaisesRegex( + watchdog.ProcfsError, "malformed MemAvailable" + ): + watchdog.ProcfsReader._parse_meminfo( + "MemTotal: 10 kB\nMemAvailable: unknown\n" + ) + with self.assertRaisesRegex( + watchdog.ProcfsError, "missing MemAvailable" + ): + watchdog.ProcfsReader._parse_meminfo("MemTotal: 10 kB\n") + with self.assertRaisesRegex( + watchdog.ProcfsError, "malformed swaps header" + ): + watchdog.ProcfsReader._parse_swaps("") + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex( + watchdog.ProcfsError, "cannot read" + ): + watchdog.ProcfsReader( + Path(temp_dir) + ).read_snapshot() + + +class TestWatchdogBehavior(unittest.TestCase): + @staticmethod + def _process_is_running(process_id: int) -> bool: + result = subprocess.run( + ["ps", "-o", "stat=", "-p", str(process_id)], + capture_output=True, + check=False, + text=True, + ) + return result.returncode == 0 and not result.stdout.lstrip().startswith( + "Z" + ) + + @staticmethod + def _write_procfs_fixture(root: Path) -> None: + (root / "meminfo").write_text( + "MemTotal: 131072 kB\nMemAvailable: 65536 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + + @staticmethod + def _lease_arguments(root: Path) -> list[str]: + return [ + "--lease-path", + str(root / "lease.json"), + "--heartbeat-path", + str(root / "heartbeat.json"), + "--audit-path", + str(root / "persistent-audit.jsonl"), + ] + + @staticmethod + def _proc_stat( + process_id: int, + parent_id: int, + process_group_id: int, + start_time_ticks: int, + ) -> str: + fields = [ + "S", + str(parent_id), + str(process_group_id), + *(["0"] * 16), + str(start_time_ticks), + ] + return f"{process_id} (python) {' '.join(fields)}\n" + + def test_parent_signals_leave_no_child_or_grandchild(self) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGHUP,signal.SIG_IGN);" + "signal.signal(signal.SIGINT,signal.SIG_IGN);" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "grandchild=os.fork();" + "\nif grandchild == 0:\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n');" + " time.sleep(30)\n" + ) + for signal_number in ( + signal.SIGHUP, + signal.SIGINT, + signal.SIGTERM, + ): + with self.subTest(signal=signal.Signals(signal_number).name): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pids" + self._write_procfs_fixture(root) + stderr_path = root / "stderr.jsonl" + with stderr_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.2", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + self.fail( + "child process group did not start" + ) + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_file.read_text( + encoding="utf-8" + ).split() + ) + time.sleep(0.05) + wrapper.send_signal(signal_number) + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + self.assertEqual( + wrapper.returncode, 128 + signal_number + ) + records = [ + json.loads(line) + for line in stderr_path.read_text( + encoding="utf-8" + ).splitlines() + ] + self.assertEqual( + records[-1]["classification"], "parent_signal" + ) + signal_records = [ + record + for record in records + if record["event"] == "process_group_signal" + ] + forwarded = [ + record["signal"] for record in signal_records + ] + self.assertEqual( + forwarded[0], + signal.Signals(signal_number).name, + ) + self.assertEqual(forwarded[-1], "SIGKILL") + self.assertEqual( + signal_records[0]["child_status"], "running" + ) + self.assertIsNone( + signal_records[0]["child_returncode"] + ) + self.assertLess( + signal_records[0]["timestamp"], + signal_records[-1]["timestamp"], + ) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse( + self._process_is_running(process_id) + ) + lease = json.loads( + (root / "lease.json").read_text( + encoding="utf-8" + ) + ) + heartbeat = json.loads( + (root / "heartbeat.json").read_text( + encoding="utf-8" + ) + ) + persistent_records = [ + json.loads(line) + for line in ( + root / "persistent-audit.jsonl" + ).read_text(encoding="utf-8").splitlines() + ] + self.assertEqual(lease["state"], "final") + self.assertEqual( + lease["final"]["classification"], + "parent_signal", + ) + self.assertEqual(heartbeat["state"], "final") + self.assertEqual( + persistent_records[-1]["classification"], + "parent_signal", + ) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_parent_signal_grace_outlives_guardian_pulse_timeout( + self, + ) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "open(sys.argv[1],'w').write(str(os.getpid()));" + "time.sleep(30)" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pid" + stderr_path = root / "stderr.jsonl" + self._write_procfs_fixture(root) + with stderr_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.4", + "--sample-interval-seconds", + "0.05", + "--heartbeat-max-age-seconds", + "0.1", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + child_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + self.fail("child process did not become ready") + time.sleep(0.01) + child_pid = int( + pid_file.read_text(encoding="utf-8") + ) + started = time.monotonic() + wrapper.send_signal(signal.SIGTERM) + wrapper.wait(timeout=5) + elapsed = time.monotonic() - started + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in stderr_path.read_text( + encoding="utf-8" + ).splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertGreaterEqual(elapsed, 0.35) + self.assertEqual( + wrapper.returncode, + 128 + signal.SIGTERM, + records, + ) + self.assertEqual(signals, ["SIGTERM", "SIGKILL"]) + self.assertEqual( + records[-1]["classification"], "parent_signal" + ) + self.assertFalse(self._process_is_running(child_pid)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_parent_signal_allows_exit_after_pulse_deadline(self) -> None: + child_code = ( + "import os,signal,sys,time\n" + "def stop(_signal,_frame):\n" + " time.sleep(0.25)\n" + " raise SystemExit(0)\n" + "signal.signal(signal.SIGTERM,stop)\n" + "open(sys.argv[1],'w').write(str(os.getpid()))\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pid" + stderr_path = root / "stderr.jsonl" + self._write_procfs_fixture(root) + with stderr_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.4", + "--sample-interval-seconds", + "0.05", + "--heartbeat-max-age-seconds", + "0.1", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + try: + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + self.fail("child process did not become ready") + time.sleep(0.01) + started = time.monotonic() + wrapper.send_signal(signal.SIGTERM) + wrapper.wait(timeout=5) + elapsed = time.monotonic() - started + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + + records = [ + json.loads(line) + for line in stderr_path.read_text( + encoding="utf-8" + ).splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertGreaterEqual(elapsed, 0.2) + self.assertLess(elapsed, 0.4) + self.assertEqual(wrapper.returncode, 128 + signal.SIGTERM) + self.assertEqual(signals, ["SIGTERM"]) + self.assertEqual(records[-1]["child_returncode"], 0) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_control_failure_still_kills_and_reaps_group( + self, + ) -> None: + class FailingFinalAudit: + def __init__(self, stream: Any, fail_at: int): + self.stream = stream + self.write_count = 0 + self.fail_at = fail_at + + def write(self, value: str) -> int: + self.write_count += 1 + if self.write_count == self.fail_at: + raise OSError("audit write failed") + return self.stream.write(value) + + def flush(self) -> None: + self.stream.flush() + + def fileno(self) -> int: + return self.stream.fileno() + + def close(self) -> None: + self.stream.close() + + class FailingFinalLease: + def finalize(self, record: dict[str, Any]) -> None: + raise watchdog.ArtifactError( + "lease", "final lease write failed" + ) + + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "grandchild=os.fork();" + "\nif grandchild == 0:\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n');" + " time.sleep(30)\n" + ) + for mode in ("closed", "blocked"): + for artifact_failure in ( + "term_audit", + "kill_audit", + "final_audit", + "lease", + ): + with self.subTest( + mode=mode, + artifact_failure=artifact_failure, + ): + self._assert_guardian_control_failure_cleanup( + mode, + artifact_failure, + child_code, + FailingFinalAudit, + FailingFinalLease, + ) + + def _assert_guardian_control_failure_cleanup( + self, + mode: str, + artifact_failure: str, + child_code: str, + failing_final_audit: type, + failing_final_lease: type, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + pid_path = Path(temp_dir) / "pids" + process = subprocess.Popen( + [ + sys.executable, + "-c", + child_code, + str(pid_path), + ], + start_new_session=True, + ) + read_fd, write_fd = os.pipe() + os.set_blocking(write_fd, False) + if mode == "closed": + os.close(write_fd) + write_fd = -1 + else: + try: + while True: + os.write(write_fd, b"x" * 65536) + except BlockingIOError: + pass + guardian = watchdog.GuardianProcess( + process, + process.pid, + write_fd, + ) + stream = io.StringIO() + audit = watchdog.AuditLogger(stream) + if artifact_failure.endswith("_audit"): + persistent_path = Path(temp_dir) / "persistent.jsonl" + persistent_stream = persistent_path.open( + "w", encoding="utf-8" + ) + audit.persistent_stream = failing_final_audit( + persistent_stream, + { + "term_audit": 1, + "kill_audit": 2, + "final_audit": 3, + }[artifact_failure], + ) + else: + audit.lease_manager = failing_final_lease() + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if time.monotonic() >= deadline: + self.fail("child process group did not start") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + result = watchdog._graceful_cleanup( + audit, + guardian, + snapshot(50), + 50, + "parent_signal", + 128 + signal.SIGTERM, + "wrapper received SIGTERM", + signal.SIGTERM, + 0.4, + watchdog._signal_process_group, + watchdog._process_group_alive, + time.monotonic, + time.sleep, + ) + finally: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + guardian.close() + os.close(read_fd) + + records = [ + json.loads(line) + for line in stream.getvalue().splitlines() + ] + signals = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual(result, watchdog.EXIT_SIGNAL_ERROR) + self.assertEqual(signals, ["SIGTERM", "SIGKILL"]) + self.assertEqual( + records[-1]["classification"], "signal_error" + ) + self.assertEqual(records[-1]["exit_code"], 7) + self.assertEqual( + records[-1]["threshold_reason"], + "guardian control failed during graceful cleanup", + ) + self.assertEqual( + records[-1]["secondary_errors"][0]["component"], + ( + "audit" + if artifact_failure.endswith("_audit") + else "lease" + ), + ) + self.assertIn( + ( + "audit write failed" + if artifact_failure.endswith("_audit") + else "final lease write failed" + ), + records[-1]["secondary_errors"][0]["detail"], + ) + self.assertEqual( + records[-1]["child_returncode"], + -signal.SIGKILL, + ) + assert child_pid is not None + assert grandchild_pid is not None + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_pipe_close_kills_group_without_fd_leak(self) -> None: + child_code = ( + "import json,os,subprocess,sys,time\n" + "targets=[]\n" + "for name in os.listdir('/proc/self/fd'):\n" + " try: targets.append(os.readlink('/proc/self/fd/'+name))\n" + " except OSError: pass\n" + "grandchild=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'])\n" + "open(sys.argv[1],'w').write(json.dumps({" + "'child':os.getpid(),'grandchild':grandchild.pid," + "'fds':targets}))\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + state_path = root / "state.json" + guardian = watchdog._launch_guardian( + ( + sys.executable, + "-c", + child_code, + str(state_path), + ), + os.environ.copy(), + 0.5, + 1.0, + signal.pthread_sigmask(signal.SIG_BLOCK, ()), + ) + control_target = os.readlink( + f"/proc/self/fd/{guardian.pulse_fd}" + ) + deadline = time.monotonic() + 5 + while not state_path.exists(): + if time.monotonic() >= deadline: + self.fail("guardian payload did not become ready") + time.sleep(0.01) + state = json.loads(state_path.read_text(encoding="utf-8")) + os.close(guardian.pulse_fd) + guardian.wait(timeout=5) + + self.assertNotIn(control_target, state["fds"]) + for process_id in (state["child"], state["grandchild"]): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_documents_setsid_escape_limit(self) -> None: + child_code = ( + "import os,subprocess,sys,time\n" + "escaped=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'],start_new_session=True)\n" + "open(sys.argv[1],'w').write(str(escaped.pid))\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + state_path = Path(temp_dir) / "escaped-pid" + guardian = watchdog._launch_guardian( + ( + sys.executable, + "-c", + child_code, + str(state_path), + ), + os.environ.copy(), + 0.5, + 1.0, + signal.pthread_sigmask(signal.SIG_BLOCK, ()), + ) + deadline = time.monotonic() + 5 + while not state_path.exists(): + if time.monotonic() >= deadline: + self.fail("escaped payload did not become ready") + time.sleep(0.01) + escaped_pid = int( + state_path.read_text(encoding="utf-8") + ) + os.close(guardian.pulse_fd) + guardian.wait(timeout=5) + self.assertTrue(self._process_is_running(escaped_pid)) + os.kill(escaped_pid, signal.SIGKILL) + deadline = time.monotonic() + 2 + while ( + self._process_is_running(escaped_pid) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(escaped_pid)) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guard_kills_group_after_watchdog_loss_or_stall(self) -> None: + child_code = ( + "import importlib.util,os,pathlib,subprocess,sys,time\n" + "script=pathlib.Path(sys.argv[1])\n" + "spec=importlib.util.spec_from_file_location('guard_watchdog',script)\n" + "module=importlib.util.module_from_spec(spec)\n" + "sys.modules[spec.name]=module\n" + "spec.loader.exec_module(module)\n" + "module.start_process_group_lease_guard(" + "script,expected_procfs_root=pathlib.Path(sys.argv[2]))\n" + "grandchild=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'])\n" + "open(sys.argv[3],'w').write(" + "f'{os.getpid()} {grandchild.pid}\\n')\n" + "time.sleep(30)\n" + ) + for mode in ("sigkill", "sigstop"): + with self.subTest(mode=mode): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_path = root / "pids" + self._write_procfs_fixture(root) + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--heartbeat-max-age-seconds", + "0.3", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(SCRIPT_PATH), + str(root), + str(pid_path), + ], + stderr=subprocess.PIPE, + text=True, + ) + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if wrapper.poll() is not None: + assert wrapper.stderr is not None + self.fail(wrapper.stderr.read()) + if time.monotonic() >= deadline: + self.fail( + "guarded payload did not become ready" + ) + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + if mode == "sigkill": + wrapper.kill() + else: + os.kill(wrapper.pid, signal.SIGSTOP) + heartbeat_path = root / "heartbeat.json" + heartbeat = json.loads( + heartbeat_path.read_text(encoding="utf-8") + ) + heartbeat["updated_monotonic_ns"] = ( + time.monotonic_ns() + ) + watchdog._write_json_atomic( + heartbeat_path, heartbeat + ) + time.sleep(0.7) + os.kill(wrapper.pid, signal.SIGCONT) + wrapper.wait(timeout=5) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse( + self._process_is_running(process_id) + ) + if wrapper.stderr is not None: + wrapper.stderr.close() + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_payload_guard_fails_closed_on_artifact_error(self) -> None: + child_code = ( + "import importlib.util,os,pathlib,subprocess,sys,time\n" + "script=pathlib.Path(sys.argv[1])\n" + "spec=importlib.util.spec_from_file_location('guard_watchdog',script)\n" + "module=importlib.util.module_from_spec(spec)\n" + "sys.modules[spec.name]=module\n" + "spec.loader.exec_module(module)\n" + "module.start_process_group_lease_guard(" + "script,expected_procfs_root=pathlib.Path(sys.argv[2]))\n" + "def fail(*_args,**_kwargs):\n" + " raise module.ArtifactError('script','unreadable')\n" + "module.validate_active_lease=fail\n" + "grandchild=subprocess.Popen([sys.executable,'-c'," + "'import time;time.sleep(30)'])\n" + "open(sys.argv[3],'w').write(" + "f'{os.getpid()} {grandchild.pid}\\n')\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_path = root / "pids" + self._write_procfs_fixture(root) + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--heartbeat-max-age-seconds", + "0.3", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(SCRIPT_PATH), + str(root), + str(pid_path), + ], + stderr=subprocess.PIPE, + text=True, + ) + child_pid = None + grandchild_pid = None + try: + deadline = time.monotonic() + 5 + while not pid_path.exists(): + if wrapper.poll() is not None: + assert wrapper.stderr is not None + self.fail(wrapper.stderr.read()) + if time.monotonic() >= deadline: + self.fail("guarded payload did not become ready") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_path.read_text( + encoding="utf-8" + ).split() + ) + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + if wrapper.stderr is not None: + wrapper.stderr.close() + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + assert child_pid is not None + assert grandchild_pid is not None + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + + def test_child_sigterm_handler_exits_without_escalation(self) -> None: + child_code = ( + "import os,signal,sys,time\n" + "def stop(_signal,_frame):\n" + " open(sys.argv[2],'w').write('handled\\n')\n" + " raise SystemExit(0)\n" + "signal.signal(signal.SIGTERM,stop)\n" + "open(sys.argv[1],'w').write(f'{os.getpid()}\\n')\n" + "time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + ready_path = root / "ready" + handled_path = root / "handled" + audit_path = root / "audit.jsonl" + self._write_procfs_fixture(root) + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--grace-seconds", + "0.5", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(ready_path), + str(handled_path), + ], + stderr=audit, + text=True, + ) + deadline = time.monotonic() + 5 + while not ready_path.exists(): + if time.monotonic() >= deadline: + wrapper.kill() + wrapper.wait(timeout=5) + self.fail("SIGTERM child did not become ready") + time.sleep(0.01) + child_pid = int( + ready_path.read_text(encoding="utf-8").strip() + ) + try: + wrapper.send_signal(signal.SIGTERM) + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual(wrapper.returncode, 128 + signal.SIGTERM) + self.assertTrue(handled_path.exists()) + self.assertEqual(forwarded, ["SIGTERM"]) + self.assertEqual(records[-1]["child_returncode"], 0) + + def test_leader_exit_cleans_up_surviving_grandchild(self) -> None: + child_code = ( + "import os,signal,sys,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "grandchild=os.fork();" + "\nif grandchild == 0:\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n')\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pids" + audit_path = root / "audit.jsonl" + self._write_procfs_fixture(root) + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--grace-seconds", + "0.2", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + wrapper.kill() + wrapper.wait(timeout=5) + self.fail("leader process did not write child PIDs") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_file.read_text( + encoding="utf-8" + ).split() + ) + try: + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + self.assertEqual(wrapper.returncode, 0) + self.assertEqual(records[-1]["classification"], "child_exit") + self.assertEqual(records[-1]["child_returncode"], 0) + self.assertEqual(forwarded, ["SIGTERM", "SIGKILL"]) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + + def test_soft_limit_descendant_escalation_is_grace_timeout(self) -> None: + child_code = ( + "import os,signal,sys,time\n" + "def stop(_signal,_frame):\n" + " raise SystemExit(0)\n" + "signal.signal(signal.SIGTERM,stop)\n" + "grandchild=os.fork()\n" + "if grandchild == 0:\n" + " signal.signal(signal.SIGTERM,signal.SIG_IGN)\n" + " time.sleep(30)\n" + "else:\n" + " open(sys.argv[1],'w').write(" + "f'{os.getpid()} {grandchild}\\n')\n" + " time.sleep(30)\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + pid_file = root / "pids" + audit_path = root / "audit.jsonl" + (root / "meminfo").write_text( + "MemTotal: 3145728 kB\n" + "MemAvailable: 2621440 kB\n", + encoding="utf-8", + ) + (root / "swaps").write_text( + "Filename Type Size Used Priority\n", + encoding="utf-8", + ) + with audit_path.open("w", encoding="utf-8") as audit: + wrapper = subprocess.Popen( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + "--soft-gib", + "1", + "--emergency-gib", + "2", + "--grace-seconds", + "0.2", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + child_code, + str(pid_file), + ], + stderr=audit, + text=True, + ) + deadline = time.monotonic() + 5 + while not pid_file.exists(): + if time.monotonic() >= deadline: + wrapper.kill() + wrapper.wait(timeout=5) + self.fail("soft-limit process group did not start") + time.sleep(0.01) + child_pid, grandchild_pid = ( + int(value) + for value in pid_file.read_text( + encoding="utf-8" + ).split() + ) + next_meminfo = root / "meminfo.next" + next_meminfo.write_text( + "MemTotal: 3145728 kB\n" + "MemAvailable: 1572864 kB\n", + encoding="utf-8", + ) + next_meminfo.replace(root / "meminfo") + try: + wrapper.wait(timeout=5) + finally: + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=5) + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + records = [ + json.loads(line) + for line in audit_path.read_text( + encoding="utf-8" + ).splitlines() + ] + forwarded = [ + record["signal"] + for record in records + if record["event"] == "process_group_signal" + ] + final = records[-1] + self.assertEqual(wrapper.returncode, watchdog.EXIT_GRACE_TIMEOUT) + self.assertEqual(final["classification"], "grace_timeout") + self.assertEqual(final["child_returncode"], 0) + self.assertEqual(forwarded, ["SIGTERM", "SIGKILL"]) + for process_id in (child_pid, grandchild_pid): + deadline = time.monotonic() + 2 + while ( + self._process_is_running(process_id) + and time.monotonic() < deadline + ): + time.sleep(0.01) + self.assertFalse(self._process_is_running(process_id)) + + def test_configuration_rejects_non_finite_timing(self) -> None: + config = watchdog.WatchdogConfig( + command=("fake-command",), + grace_seconds=float("nan"), + ) + with self.assertRaisesRegex(ValueError, "grace period"): + config.validate() + + def test_configuration_rejects_weakened_liveness_timing(self) -> None: + cases = ( + ( + {"grace_seconds": 31.0}, + "grace period", + ), + ( + {"sample_interval_seconds": 1.1}, + "sample interval", + ), + ( + { + "sample_interval_seconds": 1.0, + "heartbeat_max_age_seconds": 5.1, + }, + "heartbeat max age", + ), + ) + for overrides, message in cases: + with self.subTest(overrides=overrides): + config = watchdog.WatchdogConfig( + command=("fake-command",), + **overrides, + ) + with self.assertRaisesRegex(ValueError, message): + config.validate() + + def test_stderr_failure_does_not_bypass_cleanup(self) -> None: + class FailingStderr(io.StringIO): + def write(self, value: str) -> int: + raise OSError("stderr closed") + + process = FakeProcess() + + def exit_on_kill( + target: FakeProcess, signal_number: int + ) -> None: + if signal_number == signal.SIGKILL: + target.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50)], + process, + signal_handler=exit_on_kill, + ) + harness.audit = watchdog.AuditLogger(FailingStderr()) + with tempfile.TemporaryDirectory() as temp_dir: + persistent_path = Path(temp_dir) / "audit.jsonl" + harness.audit.open_persistent(persistent_path) + result = watchdog._graceful_cleanup( + harness.audit, + process, + snapshot(50), + 50, + "internal_error", + watchdog.EXIT_INTERNAL_ERROR, + "test cleanup", + signal.SIGTERM, + 0.1, + harness.signal_group, + harness.group_alive, + harness.clock.monotonic, + harness.clock.sleep, + ) + harness.audit.close() + records = [ + json.loads(line) + for line in persistent_path.read_text( + encoding="utf-8" + ).splitlines() + ] + + self.assertEqual( + harness.signals, [signal.SIGTERM, signal.SIGKILL] + ) + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(records[-1]["classification"], "lease_error") + + def test_audit_write_and_close_failures_do_not_bypass_cleanup( + self, + ) -> None: + class FailingPersistent(io.StringIO): + def __init__(self) -> None: + super().__init__() + self.close_called = False + + def write(self, value: str) -> int: + raise OSError("persistent write failed") + + def close(self) -> None: + if self.close_called: + super().close() + return + self.close_called = True + raise OSError("persistent close failed") + + process = FakeProcess() + + def exit_on_kill( + target: FakeProcess, signal_number: int + ) -> None: + if signal_number == signal.SIGKILL: + target.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50)], + process, + signal_handler=exit_on_kill, + ) + persistent = FailingPersistent() + harness.audit.persistent_stream = persistent + result = watchdog._graceful_cleanup( + harness.audit, + process, + snapshot(50), + 50, + "internal_error", + watchdog.EXIT_INTERNAL_ERROR, + "test cleanup", + signal.SIGTERM, + 0.1, + harness.signal_group, + harness.group_alive, + harness.clock.monotonic, + harness.clock.sleep, + ) + + self.assertTrue(persistent.close_called) + self.assertEqual( + harness.signals, [signal.SIGTERM, signal.SIGKILL] + ) + self.assertEqual(process.returncode, -signal.SIGKILL) + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual( + harness.records()[-1]["classification"], "lease_error" + ) + + def test_final_record_survives_artifact_failures(self) -> None: + class FailingLease: + def finalize(self, record: dict[str, Any]) -> None: + raise watchdog.ArtifactError("lease", "write failed") + + class FailingAudit(io.StringIO): + def write(self, value: str) -> int: + raise OSError("write failed") + + for component in ("lease", "audit"): + with self.subTest(component=component): + stream = io.StringIO() + audit = watchdog.AuditLogger(stream) + if component == "lease": + setattr(audit, "lease_manager", FailingLease()) + else: + audit.persistent_stream = FailingAudit() + result = watchdog._emit_final( + audit, + "child_exit", + 0, + "child exited", + snapshot(50), + 50, + ) + records = [ + json.loads(line) + for line in stream.getvalue().splitlines() + ] + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual( + records[-1]["classification"], "lease_error" + ) + self.assertTrue(audit.finalized) + self.assertEqual( + audit.final_exit_code, watchdog.EXIT_LEASE_ERROR + ) + + def test_emergency_signal_precedes_artifact_write(self) -> None: + events: list[str] = [] + + class BlockingAudit(watchdog.AuditLogger): + def __init__(self) -> None: + super().__init__(io.StringIO()) + self.calls = 0 + + def emit( + self, event: str, **fields: object + ) -> dict[str, object]: + self.calls += 1 + events.append(f"audit:{event}") + if self.calls == 3: + raise watchdog.ArtifactError( + "audit", "simulated blocked fsync" + ) + return super().emit(event, **fields) + + def exit_on_kill( + process: FakeProcess, signal_number: int + ) -> None: + events.append(f"signal:{signal_number}") + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), snapshot(160)], + FakeProcess(), + signal_handler=exit_on_kill, + ) + harness.audit = BlockingAudit() + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(events[2], f"signal:{signal.SIGKILL}") + self.assertEqual(events[3], "audit:process_group_signal") + self.assertEqual(harness.process.returncode, -signal.SIGKILL) + + def test_cleanup_reaps_after_persistent_audit_failure(self) -> None: + class FailingSignalAudit(watchdog.AuditLogger): + def emit( + self, event: str, **fields: object + ) -> dict[str, object]: + if event == "process_group_signal": + raise watchdog.ArtifactError( + "audit", "simulated persistent write failure" + ) + return super().emit(event, **fields) + + process = FakeProcess() + signals: list[int] = [] + clock = FakeClock() + + def signal_group( + process_group_id: int, signal_number: int + ) -> str: + signals.append(signal_number) + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + return f"{signal.Signals(signal_number).name.lower()}_sent" + + result = watchdog._graceful_cleanup( + FailingSignalAudit(io.StringIO()), + process, + snapshot(50), + 50, + "parent_signal", + 128 + signal.SIGTERM, + "wrapper received SIGTERM", + signal.SIGTERM, + 0.1, + signal_group, + lambda _process_group_id: process.returncode is None, + clock.monotonic, + clock.sleep, + ) + + self.assertEqual(result, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(signals, [signal.SIGTERM, signal.SIGKILL]) + self.assertEqual(process.returncode, -signal.SIGKILL) + + def test_invalid_artifact_path_emits_configuration_final(self) -> None: + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--lease-path", + "~strix-watchdog-user-does-not-exist/lease.json", + "--heartbeat-path", + "/tmp/heartbeat.json", + "--audit-path", + "/tmp/audit.jsonl", + "--", + sys.executable, + "-c", + "pass", + ], + capture_output=True, + check=False, + text=True, + ) + + self.assertEqual(result.returncode, watchdog.EXIT_PROCFS_ERROR) + final = json.loads(result.stderr.splitlines()[-1]) + self.assertEqual(final["classification"], "configuration_error") + + def test_cli_fixture_launches_command_and_propagates_exit(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + child_result_path = root / "child-result.json" + self._write_procfs_fixture(root) + child_code = ( + "import json,os,sys,time\n" + "keys=('STRIX_MEMORY_WATCHDOG_LEASE_PATH'," + "'STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH'," + "'STRIX_MEMORY_WATCHDOG_AUDIT_PATH')\n" + "deadline=time.monotonic()+5\n" + "while True:\n" + " try:\n" + " with open(os.environ[keys[0]],encoding='utf-8') as stream:\n" + " lease=json.load(stream)\n" + " break\n" + " except (OSError,json.JSONDecodeError):\n" + " if time.monotonic()>=deadline: raise\n" + " time.sleep(0.01)\n" + "assert os.getpgrp()==lease['child_process_group_id']\n" + "open(sys.argv[1],'w').write(json.dumps({" + "key:os.environ[key] for key in keys}))\n" + "raise SystemExit(23)\n" + ) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--sample-interval-seconds", + "0.01", + "--", + sys.executable, + "-c", + child_code, + str(child_result_path), + ], + capture_output=True, + check=False, + text=True, + ) + lease = json.loads( + (root / "lease.json").read_text(encoding="utf-8") + ) + persistent_records = [ + json.loads(line) + for line in ( + root / "persistent-audit.jsonl" + ).read_text(encoding="utf-8").splitlines() + ] + child_result = json.loads( + child_result_path.read_text(encoding="utf-8") + ) + + self.assertEqual(result.returncode, 23) + records = [ + json.loads(line) for line in result.stderr.splitlines() + ] + self.assertEqual(records[-1]["classification"], "child_exit") + self.assertEqual(records[-1]["child_returncode"], 23) + self.assertEqual(lease["format"], watchdog.LEASE_FORMAT) + self.assertEqual(lease["version"], watchdog.LEASE_VERSION) + self.assertEqual(lease["state"], "final") + self.assertEqual(lease["soft_bytes"], 116 * 1024**3) + self.assertEqual( + lease["emergency_bytes"], 118 * 1024**3 + ) + self.assertEqual( + lease["child_command_sha256"], + watchdog._command_sha256( + ( + sys.executable, + "-c", + child_code, + str(child_result_path), + ) + ), + ) + self.assertEqual( + lease["final"]["classification"], "child_exit" + ) + self.assertEqual( + persistent_records[-1]["classification"], "child_exit" + ) + self.assertEqual( + child_result["STRIX_MEMORY_WATCHDOG_LEASE_PATH"], + str((root / "lease.json").resolve()), + ) + self.assertEqual( + child_result["STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH"], + str((root / "heartbeat.json").resolve()), + ) + self.assertEqual( + child_result["STRIX_MEMORY_WATCHDOG_AUDIT_PATH"], + str((root / "persistent-audit.jsonl").resolve()), + ) + + @unittest.skipUnless( + sys.platform.startswith("linux"), + "Linux guardian lifecycle", + ) + def test_guardian_preserves_payload_signal_status(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self._write_procfs_fixture(root) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--sample-interval-seconds", + "0.01", + "--", + sys.executable, + "-c", + ( + "import os,signal,time;" + "time.sleep(0.1);" + "os.kill(os.getpid(),signal.SIGTERM)" + ), + ], + capture_output=True, + check=False, + text=True, + timeout=5, + ) + records = [ + json.loads(line) + for line in result.stderr.splitlines() + ] + + self.assertEqual(result.returncode, 128 + signal.SIGTERM) + self.assertEqual(records[-1]["classification"], "child_exit") + self.assertEqual( + records[-1]["child_returncode"], -signal.SIGTERM + ) + self.assertEqual(records[-1]["child_status"], "signaled") + + def test_existing_lease_fails_closed_and_stops_child(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self._write_procfs_fixture(root) + lease_path = root / "lease.json" + lease_path.write_text("untrusted\n", encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--procfs-root", + str(root), + *self._lease_arguments(root), + "--grace-seconds", + "0.1", + "--sample-interval-seconds", + "0.05", + "--", + sys.executable, + "-c", + "import time;time.sleep(30)", + ], + capture_output=True, + check=False, + text=True, + timeout=5, + ) + records = [ + json.loads(line) for line in result.stderr.splitlines() + ] + child_pid = records[-1]["child_pid"] + self.assertEqual( + lease_path.read_text(encoding="utf-8"), "untrusted\n" + ) + + self.assertEqual(result.returncode, watchdog.EXIT_LEASE_ERROR) + self.assertEqual(records[-1]["classification"], "lease_error") + self.assertFalse(self._process_is_running(child_pid)) + + def test_active_lease_validation_rejects_tamper_and_stale(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + process_root = root / "proc" + watchdog_pid = 1200 + guardian_pid = 1250 + child_pid = 1300 + current_pid = 1400 + watchdog_start_ticks = 456789 + script_path = root / "watchdog.py" + script_path.write_text("print('watchdog')\n", encoding="utf-8") + lease_path = root / "lease.json" + heartbeat_path = root / "heartbeat.json" + audit_path = root / "audit.jsonl" + command = [sys.executable, "run_matrix.py"] + argv = [ + sys.executable, + "watchdog.py", + "--procfs-root", + "/proc", + "--lease-path", + str(lease_path), + "--heartbeat-path", + str(heartbeat_path), + "--audit-path", + str(audit_path), + "--", + *command, + ] + cmdline = b"\0".join(os.fsencode(value) for value in argv) + for process_id, parent_id, group_id, start_ticks in ( + (watchdog_pid, 1, watchdog_pid, watchdog_start_ticks), + (guardian_pid, watchdog_pid, guardian_pid, 456790), + (child_pid, guardian_pid, guardian_pid, 456791), + (current_pid, child_pid, guardian_pid, 456792), + ): + process_dir = process_root / str(process_id) + process_dir.mkdir(parents=True) + (process_dir / "stat").write_text( + self._proc_stat( + process_id, + parent_id, + group_id, + start_ticks, + ), + encoding="utf-8", + ) + (process_root / str(watchdog_pid) / "cwd").symlink_to( + root, target_is_directory=True + ) + (process_root / str(watchdog_pid) / "exe").symlink_to( + Path(sys.executable).resolve() + ) + (process_root / str(watchdog_pid) / "cmdline").write_bytes( + cmdline + ) + + audit_line = ( + '{"event":"child_started",' + '"timestamp":"2026-01-01T00:00:00Z"}\n' + ) + audit_descriptor = os.open( + audit_path, + os.O_CREAT | os.O_EXCL | os.O_RDWR, + 0o600, + ) + os.write(audit_descriptor, audit_line.encode("utf-8")) + os.fsync(audit_descriptor) + fcntl.flock( + audit_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB + ) + audit_status = os.fstat(audit_descriptor) + fd_root = process_root / str(watchdog_pid) / "fd" + fd_root.mkdir() + (fd_root / "9").symlink_to(audit_path) + lease = { + "format": watchdog.LEASE_FORMAT, + "version": watchdog.LEASE_VERSION, + "lease_id": "test-lease", + "state": "active", + "watchdog_pid": watchdog_pid, + "watchdog_start_time_utc": "2026-01-01T00:00:00.000Z", + "watchdog_start_time_ticks": watchdog_start_ticks, + "watchdog_command_sha256": hashlib.sha256( + cmdline + ).hexdigest(), + "watchdog_executable_path": str( + Path(sys.executable).resolve() + ), + "watchdog_script_path": str(script_path), + "watchdog_script_sha256": hashlib.sha256( + script_path.read_bytes() + ).hexdigest(), + "soft_bytes": watchdog.DEFAULT_SOFT_BYTES, + "emergency_bytes": watchdog.DEFAULT_EMERGENCY_BYTES, + "strict_ceiling_bytes": watchdog.STRICT_CEILING_BYTES, + "grace_seconds": watchdog.DEFAULT_GRACE_SECONDS, + "sample_interval_seconds": ( + watchdog.DEFAULT_SAMPLE_INTERVAL_SECONDS + ), + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": guardian_pid, + "command": command, + "child_command_sha256": watchdog._command_sha256( + command + ), + "heartbeat_path": str(heartbeat_path), + "max_heartbeat_age_seconds": 5.0, + "audit_path": str(audit_path), + "audit_device": audit_status.st_dev, + "audit_inode": audit_status.st_ino, + "audit_uid": audit_status.st_uid, + "audit_mode": 0o600, + "audit_fd": 9, + "procfs_root": "/proc", + } + heartbeat = { + "format": watchdog.HEARTBEAT_FORMAT, + "version": watchdog.HEARTBEAT_VERSION, + "lease_id": "test-lease", + "sequence": 4, + "state": "active", + "updated_at": "2026-01-01T00:00:01.000Z", + "updated_monotonic_ns": 9_000_000_000, + "watchdog_pid": watchdog_pid, + "watchdog_start_time_ticks": ( + watchdog_start_ticks + ), + "child_pid": child_pid, + "child_process_group_id": guardian_pid, + "sample": { + "audit_record_sha256": hashlib.sha256( + audit_line.encode("utf-8") + ).hexdigest() + }, + } + watchdog._write_json_atomic( + lease_path, lease, create=True + ) + watchdog._write_json_atomic( + heartbeat_path, heartbeat, create=True + ) + + try: + validation_args = { + "expected_script_path": script_path, + "expected_executable_path": Path(sys.executable), + "expected_command": command, + "expected_heartbeat_path": heartbeat_path, + "expected_audit_path": audit_path, + "expected_max_heartbeat_age_seconds": 5.0, + "current_process_id": current_pid, + "process_procfs_root": process_root, + "monotonic_ns": lambda: 10_000_000_000, + "pidfd_open": lambda _pid: os.open( + os.devnull, os.O_RDONLY + ), + } + validated = watchdog.validate_active_lease( + lease_path, **validation_args + ) + self.assertEqual(validated["lease_id"], "test-lease") + + def publish_lease( + value: dict[str, Any], + process_argv: list[str] = argv, + ) -> None: + process_cmdline = b"\0".join( + os.fsencode(argument) + for argument in process_argv + ) + (process_root / str(watchdog_pid) / "cmdline").write_bytes( + process_cmdline + ) + value["watchdog_command_sha256"] = hashlib.sha256( + process_cmdline + ).hexdigest() + watchdog._write_json_atomic(lease_path, value) + + for name, bad_argv in ( + ( + "helper inert argument", + [ + sys.executable, + "helper.py", + str(script_path), + *argv[2:], + ], + ), + ( + "python command string", + [ + sys.executable, + "-c", + "pass", + str(script_path), + *argv[2:], + ], + ), + ( + "python module", + [ + sys.executable, + "-m", + "helper", + str(script_path), + *argv[2:], + ], + ), + ( + "interpreter option before script", + [ + sys.executable, + "-O", + str(script_path), + *argv[2:], + ], + ), + ): + with self.subTest(name): + publish_lease(dict(lease), bad_argv) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "executable argv position", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("wrong command-line policy"): + bad_argv = list(argv) + procfs_index = bad_argv.index("--procfs-root") + 1 + bad_argv[procfs_index] = "/tmp/not-proc" + publish_lease(dict(lease), bad_argv) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "command-line policy", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("wrong lease timing policy"): + bad_lease = dict(lease) + bad_lease["grace_seconds"] = 29.0 + publish_lease(bad_lease) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "lease timing policy", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("wrong monitored command"): + bad_argv = [*argv[:-1], "other_matrix.py"] + bad_lease = dict(lease) + bad_lease["command"] = [ + sys.executable, + "other_matrix.py", + ] + bad_lease["child_command_sha256"] = ( + watchdog._command_sha256( + bad_lease["command"] + ) + ) + publish_lease(bad_lease, bad_argv) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "monitored command", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("tampered script SHA"): + tampered = dict(lease) + tampered["watchdog_script_sha256"] = "0" * 64 + publish_lease(tampered) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, "script SHA" + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("stale heartbeat"): + publish_lease(dict(lease)) + heartbeat["updated_monotonic_ns"] = 1 + watchdog._write_json_atomic( + heartbeat_path, heartbeat + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "heartbeat is stale", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("arbitrary heartbeat"): + heartbeat["updated_monotonic_ns"] = 9_000_000_000 + heartbeat["lease_id"] = "helper-lease" + watchdog._write_json_atomic( + heartbeat_path, heartbeat + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "heartbeat identity", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("outside process group"): + heartbeat["lease_id"] = "test-lease" + watchdog._write_json_atomic( + heartbeat_path, heartbeat + ) + (process_root / str(current_pid) / "stat").write_text( + self._proc_stat( + current_pid, + child_pid, + 9999, + 456792, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "outside the monitored process group", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + ( + process_root / str(current_pid) / "stat" + ).write_text( + self._proc_stat( + current_pid, + child_pid, + guardian_pid, + 456792, + ), + encoding="utf-8", + ) + + with self.subTest("environment path mismatch"): + bad_validation_args = { + **validation_args, + "expected_heartbeat_path": root / "other.json", + } + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "artifact paths|heartbeat path", + ): + watchdog.validate_active_lease( + lease_path, **bad_validation_args + ) + + with self.subTest("lease inode mismatch"): + publish_lease(dict(lease)) + lease_record = json.loads( + lease_path.read_text(encoding="utf-8") + ) + lease_record["file_inode"] = 0 + lease_path.write_text( + json.dumps(lease_record), encoding="utf-8" + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "identity does not match", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + + with self.subTest("watchdog start tick mismatch"): + publish_lease(dict(lease)) + (process_root / str(watchdog_pid) / "stat").write_text( + self._proc_stat( + watchdog_pid, + 1, + watchdog_pid, + watchdog_start_ticks + 1, + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + watchdog.LeaseValidationError, + "start time", + ): + watchdog.validate_active_lease( + lease_path, **validation_args + ) + finally: + os.close(audit_descriptor) + + def test_zero_swap_gate_launches_and_propagates_child_exit(self) -> None: + harness = Harness([snapshot(50)], FakeProcess(returncode=37)) + + result = harness.run() + + self.assertEqual(result, 37) + self.assertTrue(harness.launched) + self.assertTrue(harness.launch_kwargs["start_new_session"]) + final = harness.records()[-1] + self.assertEqual(final["classification"], "child_exit") + self.assertEqual(final["total_bytes"], 200) + self.assertEqual(final["available_bytes"], 150) + self.assertEqual(final["used_bytes"], 50) + self.assertEqual(final["peak_used_bytes"], 50) + self.assertEqual(final["child_status"], "exited") + self.assertEqual(final["process_group_status"], "leader_exited") + + def test_signaled_child_exit_uses_shell_exit_convention(self) -> None: + harness = Harness( + [snapshot(50)], + FakeProcess(returncode=-signal.SIGTERM), + ) + + result = harness.run() + + self.assertEqual(result, 128 + signal.SIGTERM) + + def test_active_swap_rejects_startup_without_launch(self) -> None: + harness = Harness( + [snapshot(50, active_swaps=("/swapfile",))], + FakeProcess(), + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_SWAP_ACTIVE) + self.assertFalse(harness.launched) + self.assertEqual( + harness.records()[-1]["classification"], + "startup_swap_active", + ) + + def test_soft_limit_sends_sigterm(self) -> None: + def exit_on_term(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGTERM: + process.returncode = -signal.SIGTERM + + harness = Harness( + [snapshot(50), snapshot(110)], + FakeProcess(), + signal_handler=exit_on_term, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + result = harness.run( + lease_path=root / "lease.json", + heartbeat_path=root / "heartbeat.json", + audit_path=root / "audit.jsonl", + ) + harness.audit.close() + heartbeat = json.loads( + (root / "heartbeat.json").read_text(encoding="utf-8") + ) + lease = json.loads( + (root / "lease.json").read_text(encoding="utf-8") + ) + + self.assertEqual(result, watchdog.EXIT_SOFT_LIMIT) + self.assertEqual(harness.signals, [signal.SIGTERM]) + self.assertEqual( + harness.records()[-1]["classification"], "soft_limit" + ) + self.assertEqual(heartbeat["state"], "final") + self.assertEqual(heartbeat["sequence"], 3) + self.assertEqual(lease["final"]["classification"], "soft_limit") + + def test_emergency_limit_sends_sigkill(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), snapshot(160)], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_EMERGENCY_LIMIT) + self.assertEqual(harness.signals, [signal.SIGKILL]) + self.assertEqual( + harness.records()[-1]["classification"], "emergency_limit" + ) + + def test_grace_timeout_escalates_to_sigkill(self) -> None: + def ignore_term(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), snapshot(110)], + FakeProcess(), + signal_handler=ignore_term, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_GRACE_TIMEOUT) + self.assertEqual( + harness.signals, + [signal.SIGTERM, signal.SIGKILL], + ) + self.assertEqual(harness.clock.value, 2.0) + self.assertEqual( + harness.records()[-1]["classification"], "grace_timeout" + ) + + def test_swap_appearing_during_execution_kills_group(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [ + snapshot(50), + snapshot(60, active_swaps=("/swapfile",)), + ], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_SWAP_ACTIVE) + self.assertEqual(harness.signals, [signal.SIGKILL]) + self.assertEqual( + harness.records()[-1]["classification"], "swap_appeared" + ) + + def test_runtime_procfs_error_kills_group(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), watchdog.ProcfsError("missing meminfo")], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_PROCFS_ERROR) + self.assertEqual(harness.signals, [signal.SIGKILL]) + self.assertEqual( + harness.records()[-1]["classification"], "procfs_error" + ) + + def test_unexpected_monitor_error_cleans_up_process_group(self) -> None: + def exit_on_kill(process: FakeProcess, signal_number: int) -> None: + if signal_number == signal.SIGKILL: + process.returncode = -signal.SIGKILL + + harness = Harness( + [snapshot(50), RuntimeError("unexpected")], + FakeProcess(), + signal_handler=exit_on_kill, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + result = harness.run( + lease_path=root / "lease.json", + heartbeat_path=root / "heartbeat.json", + audit_path=root / "audit.jsonl", + ) + harness.audit.close() + lease = json.loads( + (root / "lease.json").read_text(encoding="utf-8") + ) + persistent_records = [ + json.loads(line) + for line in (root / "audit.jsonl").read_text( + encoding="utf-8" + ).splitlines() + ] + + self.assertEqual(result, watchdog.EXIT_INTERNAL_ERROR) + self.assertEqual( + harness.signals, + [signal.SIGTERM, signal.SIGKILL], + ) + final = harness.records()[-1] + self.assertEqual(final["classification"], "internal_error") + self.assertIn("RuntimeError: unexpected", final["error"]) + self.assertEqual(lease["final"]["classification"], "internal_error") + self.assertEqual( + persistent_records[-1]["classification"], "internal_error" + ) + + def test_launch_failure_is_explicit(self) -> None: + harness = Harness([snapshot(50)], FakeProcess()) + + def fail_launch( + command: tuple[str, ...], **kwargs: Any + ) -> FakeProcess: + raise FileNotFoundError(2, "No such file or directory") + + setattr(harness, "launcher", fail_launch) + + result = harness.run() + + self.assertEqual(result, watchdog.EXIT_LAUNCH_ERROR) + self.assertEqual( + harness.records()[-1]["classification"], "launch_error" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index c8ad1db43623..1cf2d8720271 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -38,6 +38,7 @@ else() add_subdirectory(export-lora) endif() add_subdirectory(fit-params) + add_subdirectory(deepseek-v41-trace) if (GGML_METAL) add_subdirectory(tuning) endif() diff --git a/tools/deepseek-v41-trace/CMakeLists.txt b/tools/deepseek-v41-trace/CMakeLists.txt new file mode 100644 index 000000000000..202c008e16fb --- /dev/null +++ b/tools/deepseek-v41-trace/CMakeLists.txt @@ -0,0 +1,252 @@ +set(TARGET llama-deepseek-v41-trace) +set(DSV41_INSTALL_COMPONENT DeepSeekV41Trace) + +if(NOT BUILD_SHARED_LIBS) + message(STATUS "Skipping DeepSeek V4.1 trace tools because BUILD_SHARED_LIBS is disabled") + return() +endif() + +find_package(Git REQUIRED) +find_package(Python3 3.10 REQUIRED COMPONENTS Interpreter) +execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE DSV41_BUILD_REVISION + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE DSV41_BUILD_REVISION_RESULT) +string(LENGTH "${DSV41_BUILD_REVISION}" DSV41_BUILD_REVISION_LENGTH) +if(NOT DSV41_BUILD_REVISION_RESULT EQUAL 0 OR + NOT DSV41_BUILD_REVISION_LENGTH EQUAL 40 OR + NOT DSV41_BUILD_REVISION MATCHES "^[0-9a-f]+$") + message(FATAL_ERROR "DeepSeek V4.1 trace exporter requires the exact Git revision") +endif() +set(DSV41_RECEIPT_TARGETS llama-common llama ggml ggml-base) +set(DSV41_BACKEND_TARGETS ${GGML_AVAILABLE_BACKENDS}) +list(SORT DSV41_BACKEND_TARGETS) +foreach(backend IN LISTS DSV41_BACKEND_TARGETS) + if(TARGET ${backend}) + list(APPEND DSV41_RECEIPT_TARGETS ${backend}) + endif() +endforeach() +list(REMOVE_DUPLICATES DSV41_RECEIPT_TARGETS) + +set(CONTAINMENT_HELPER_TARGET llama-deepseek-v41-containment-helper) +add_executable(${CONTAINMENT_HELPER_TARGET} linux-containment-helper.cpp) +target_compile_features(${CONTAINMENT_HELPER_TARGET} PRIVATE cxx_std_17) +target_compile_definitions( + ${CONTAINMENT_HELPER_TARGET} + PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") +set(DSV41_CONTAINMENT_HELPER_RECEIPT + "${CMAKE_CURRENT_BINARY_DIR}/dsv41-containment-helper-receipt.json") +add_custom_command( + OUTPUT "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + COMMAND + ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/generate-containment-helper-receipt.py" + --output "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + --helper "$" + --revision "${DSV41_BUILD_REVISION}" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/generate-containment-helper-receipt.py" + ${CONTAINMENT_HELPER_TARGET} + VERBATIM) +add_custom_target( + dsv41-containment-helper-receipt + DEPENDS "${DSV41_CONTAINMENT_HELPER_RECEIPT}") + +if(WIN32) + set(DSV41_RUNTIME_PROFILE co-located) +else() + set(DSV41_RUNTIME_PROFILE sibling-lib) + if(APPLE) + set(DSV41_LIBRARY_RPATH "@loader_path") + set(DSV41_EXECUTABLE_RPATH "@loader_path/../lib") + else() + set(DSV41_LIBRARY_RPATH "\$ORIGIN") + set(DSV41_EXECUTABLE_RPATH "\$ORIGIN/../lib") + endif() +endif() + +set(DSV41_RECEIPT_ARGUMENTS) +set(DSV41_RECEIPT_FILES) +foreach(component IN LISTS DSV41_RECEIPT_TARGETS) + if(NOT TARGET ${component}) + message(FATAL_ERROR "DeepSeek V4.1 receipt target is missing: ${component}") + endif() + get_target_property(component_type ${component} TYPE) + if(NOT component_type STREQUAL "SHARED_LIBRARY" AND NOT component_type STREQUAL "MODULE_LIBRARY") + message(FATAL_ERROR "DeepSeek V4.1 receipt target is not shared: ${component}") + endif() + if(component STREQUAL "llama-common" OR component STREQUAL "ggml-base") + set(component_revision "${DSV41_BUILD_REVISION}") + else() + set(component_revision "-") + endif() + list(APPEND DSV41_RECEIPT_ARGUMENTS + --entry "${component}" "$" "${component_revision}") + list(APPEND DSV41_RECEIPT_FILES "$") + if(NOT WIN32) + set_target_properties( + ${component} + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_LIBRARY_RPATH}") + if(APPLE) + set_target_properties( + ${component} + PROPERTIES + BUILD_WITH_INSTALL_NAME_DIR TRUE + INSTALL_NAME_DIR "@rpath") + endif() + endif() +endforeach() + +set(DSV41_RECEIPT_HEADER "${CMAKE_CURRENT_BINARY_DIR}/dsv41-runtime-receipt.h") +set(DSV41_RECEIPT_JSON "${CMAKE_CURRENT_BINARY_DIR}/dsv41-runtime-receipt.json") +add_custom_command( + OUTPUT "${DSV41_RECEIPT_HEADER}" "${DSV41_RECEIPT_JSON}" + COMMAND + ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/generate-runtime-receipt.py" + --output "${DSV41_RECEIPT_HEADER}" + --json-output "${DSV41_RECEIPT_JSON}" + --revision "${DSV41_BUILD_REVISION}" + --profile "${DSV41_RUNTIME_PROFILE}" + ${DSV41_RECEIPT_ARGUMENTS} + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/generate-runtime-receipt.py" + ${DSV41_RECEIPT_FILES} + VERBATIM) +add_custom_target( + dsv41-runtime-receipt + DEPENDS "${DSV41_RECEIPT_HEADER}" "${DSV41_RECEIPT_JSON}") +set_source_files_properties( + llama-trace.cpp + PROPERTIES OBJECT_DEPENDS "${DSV41_RECEIPT_HEADER}") + +add_executable(${TARGET} llama-trace.cpp) +add_dependencies(${TARGET} dsv41-runtime-receipt dsv41-containment-helper-receipt) +target_link_libraries(${TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) +target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) +target_compile_definitions(${TARGET} PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") +if(NOT WIN32) + set_target_properties( + ${TARGET} + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_EXECUTABLE_RPATH}") +endif() + +set(PROMPT_TARGET llama-deepseek-v41-prompt-builder) +add_executable(${PROMPT_TARGET} prompt-builder.cpp) +add_dependencies(${PROMPT_TARGET} dsv41-runtime-receipt dsv41-containment-helper-receipt) +target_link_libraries(${PROMPT_TARGET} PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) +target_include_directories(${PROMPT_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_compile_features(${PROMPT_TARGET} PRIVATE cxx_std_17) +target_compile_definitions(${PROMPT_TARGET} PRIVATE DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}") +if(NOT WIN32) + set_target_properties( + ${PROMPT_TARGET} + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_EXECUTABLE_RPATH}") +endif() + +if(LLAMA_TOOLS_INSTALL) + install( + TARGETS ${CONTAINMENT_HELPER_TARGET} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install( + FILES "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + DESTINATION "share/deepseek-v41-trace" + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ GROUP_READ WORLD_READ) + install( + TARGETS ${TARGET} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install( + TARGETS ${DSV41_RECEIPT_TARGETS} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + NAMELINK_COMPONENT ${DSV41_INSTALL_COMPONENT}) + install( + TARGETS ${PROMPT_TARGET} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT ${DSV41_INSTALL_COMPONENT} + PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +endif() + +if(LLAMA_BUILD_TESTS) + set(MANIFEST_TEST_TARGET test-deepseek41-trace-manifest) + add_executable(${MANIFEST_TEST_TARGET} llama-trace.cpp) + add_dependencies( + ${MANIFEST_TEST_TARGET} + dsv41-runtime-receipt + dsv41-containment-helper-receipt) + target_link_libraries( + ${MANIFEST_TEST_TARGET} + PRIVATE llama-common llama vendor::hash ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + target_include_directories( + ${MANIFEST_TEST_TARGET} + PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) + target_compile_features(${MANIFEST_TEST_TARGET} PRIVATE cxx_std_17) + target_compile_definitions( + ${MANIFEST_TEST_TARGET} + PRIVATE + DSV41_BUILD_REVISION="${DSV41_BUILD_REVISION}" + DSV41_SOURCE_ROOT="${CMAKE_SOURCE_DIR}" + DSV41_MANIFEST_TEST_HARNESS=1) + if(NOT WIN32) + set_target_properties( + ${MANIFEST_TEST_TARGET} + PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "${DSV41_EXECUTABLE_RPATH}") + endif() + + set(INJECTED_TEST_TARGET test-deepseek41-trace-injected) + add_library(${INJECTED_TEST_TARGET} SHARED test-injected-library.cpp) + set_target_properties( + ${INJECTED_TEST_TARGET} + PROPERTIES + PREFIX "" + OUTPUT_NAME "libggml-injected" + SUFFIX ".module") + set(HOST_TEST_TARGET test-deepseek41-trace-host) + add_executable(${HOST_TEST_TARGET} test-host-attestation.cpp) + target_link_libraries(${HOST_TEST_TARGET} PRIVATE ggml) + target_compile_features(${HOST_TEST_TARGET} PRIVATE cxx_std_17) + add_test(NAME ${HOST_TEST_TARGET} COMMAND $) + set_property(TEST ${HOST_TEST_TARGET} PROPERTY LABELS main) + + if(LLAMA_TOOLS_INSTALL) + add_test( + NAME test-deepseek41-trace-install + COMMAND + ${CMAKE_COMMAND} + -DDSV41_BUILD_DIR=${CMAKE_BINARY_DIR} + -DDSV41_INSTALL_ROOT=${CMAKE_CURRENT_BINARY_DIR}/install-smoke + -DDSV41_COMPONENT=${DSV41_INSTALL_COMPONENT} + -DDSV41_EXECUTABLE=${TARGET}${CMAKE_EXECUTABLE_SUFFIX} + -DDSV41_SOURCE_EXECUTABLE=$ + -DDSV41_REVISION=${DSV41_BUILD_REVISION} + -DDSV41_CONFIG=$ + -DDSV41_PYTHON=${Python3_EXECUTABLE} + -DDSV41_RECEIPT=${DSV41_RECEIPT_JSON} + -DDSV41_CONTAINMENT_HELPER=$ + -DDSV41_CONTAINMENT_HELPER_RECEIPT=${DSV41_CONTAINMENT_HELPER_RECEIPT} + -DDSV41_VERIFY_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/verify-runtime-install.py + -P ${CMAKE_CURRENT_SOURCE_DIR}/test-install.cmake) + set_property(TEST test-deepseek41-trace-install PROPERTY LABELS main) + endif() +endif() diff --git a/tools/deepseek-v41-trace/README.md b/tools/deepseek-v41-trace/README.md new file mode 100644 index 000000000000..bca7e6b56591 --- /dev/null +++ b/tools/deepseek-v41-trace/README.md @@ -0,0 +1,366 @@ +# DeepSeek V4.1 correctness traces + +This directory defines version 2 of the cross-runtime trace format used by issue #48. It compares the unchanged published GGUF between llama.cpp and ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. + +Each trace is a directory: + +- `manifest.json` records the model and prompt SHA-256 values, exact runtime revision/build, canonical executable path, loaded runtime-library paths and hashes, inference configuration, environment, runtime-specific host evidence, exact execution paths, and content-addressed audit references. +- `events.jsonl` is an ordered stream of content-addressed event records. +- `blobs/.bin` stores canonical little-endian tensor bytes. This keeps complete logits and per-token state exact without embedding large numeric arrays in JSON. +- `audits/pre/.json` and `audits/post/.json` store immutable safety evidence from both sides of execution. +- `provenance/.json` binds the exact prompt to its fixed corpus, published model, target token count, and prompt-builder executable. +- `bundle-signature.json` contains the detached OpenSSH signature envelope for the exact bundle file set. + +Seal v1 signs `dsv41-trace-bundle-v1\n` followed by canonical JSON records for `manifest.json`, `events.jsonl`, every referenced event blob, every embedded audit, and prompt provenance. Each record binds its canonical relative path, byte count, and SHA-256. Validation rejects missing or added files, symlinks, hard links, nonregular files, path traversal, duplicate metadata references, noncanonical JSON or JSONL, duplicate JSON keys, truncation, concurrent replacement, and every post-seal mutation. + +Signer trust is external to the bundle. `APPROVED_TRACE_SIGNERS` maps one restricted ASCII principal to one exact OpenSSH Ed25519 public key, runtime lane, and runtime profile, and is intentionally empty until a separately authorized run. Candidate and oracle signers are not interchangeable. Every signed manifest also binds an externally supplied 256-bit challenge, lane-specific run ID, and bounded validity window. Validation requires those expected values from outside the bundle and rejects missing, mismatched, reused within a comparison, cross-lane, not-yet-valid, or expired authorization. The bundle cannot provide a public key, authoritative principal, verifier path, or allowed-signers file. Validation uses only `/usr/bin/ssh-keygen` on macOS and Linux or `C:\Windows\System32\OpenSSH\ssh-keygen.exe` on Windows, rejects symlinks and unsupported `-Y` implementations, clears SSH-agent influence, and never searches `PATH`. The private signing key must be an owned restrictive regular file outside the bundle and is never copied or logged. Tests use explicit test-only verifiers with ephemeral lane-specific keys; production validation does not trust those keys. + +Executable trust is also external to the bundle and to the reviewed source revision. `APPROVED_EXECUTABLE_APPROVERS`, `APPROVED_CANDIDATE_EXPORTERS`, and `APPROVED_PROMPT_BUILDERS` are intentionally empty in production source. An authorized run must receive a canonical detached approval policy plus its OpenSSH signature and an externally expected approver principal. Both files must be absolute canonical non-symlinked one-link regular files outside every protected output root. Their trusted root and owner come only from `APPROVED_EXECUTABLE_APPROVERS`; the fixed `ssh-keygen` verifies namespace `dsv41-executable-approval-v2` before either executable can run. This avoids an impossible same-revision self-reference: the policy records the artifact producer revision and exact executable hashes, while a separate `verifier_revision` records the harness revision that consumes the policy. + +The candidate exporter approval binds the exact producer revision, base revision, binary diff SHA-256, canonical install root and exporter path, exporter SHA-256, runtime profile, complete embedded runtime receipt, and the exact revision, filename, and SHA-256 of the native Linux containment helper. The prompt-builder approval binds the same helper identity with its producer revision, canonical install/source roots and executable path, executable SHA-256, exact runtime profile and receipt, model and corpus identities, complete tokenizer policy, and exact prompt hash, byte count, context, and decode configuration for every authorized case. The tokenizer policy explicitly binds `add_bos`, `parse_special`, `detokenize_special`, leading-BOS removal, and exact token round-trip. The builder invocation, native result, signed provenance, execution authorization, candidate environment, and candidate manifest must all agree with it. + +Every approved install tree is verified before launch. The install owner is externally selected and must differ from the unprivileged execution identity. Every ancestor is canonical, non-symlinked, trusted-owned, ACL-free, and not writable by the execution identity or group/other. Every approved executable, containment helper, and receipt library is a canonical, ACL-free, non-writable, one-link regular file with the exact owner and SHA-256. On Linux, the exporter, prompt builder, and containment helper are opened without following the final symlink and launched through retained `/proc/self/fd` descriptors. Receipt-library descriptors remain open through process completion, while the immutable canonical install hierarchy prevents pathname substitution during loader consumption. Both native tools report the actual canonical loaded project-library paths and hashes; the prompt provenance and candidate manifest require the loaded closure to equal the approved receipt before and after protected work. Ordinary user-owned install smoke verifies layout, RPATH, helper receipt, version, and bytes only; it does not establish production trusted-root authorization. + +The llama runner opens the approved model once without following aliases, hashes that held read-only descriptor before launch, retains it through Linux containment, and directs the exporter to load through `/proc/self/fd`. The runner and exporter bind device, inode, owner, mode, link count, size, timestamps, descriptor flags, and SHA-256 before model initialization and verify the same descriptor plus the original pathname after tracing. The watchdog is validated against host `/proc` before containment and represented inside the private PID namespace by a retained live pidfd. The signed audit binds the host PID, process group, start time, executable, command, guardian, and child identities, while the candidate manifest separately requires the exporter to be namespace-local PID 2 with parent PID 1, its own process group and session, matching NSpid, and private procfs. Host numeric PIDs are never interpreted as namespace-local PIDs. + +Prompt provenance version 2 records the lexical and resolved source root and corpus paths. The prompt builder accepts path aliases only when both resolve to the exact approved repository corpus and binds the canonical source identity into the signed provenance. + +The signed execution authorization records the complete approval-policy SHA-256, verifier revision, tokenizer-policy SHA-256, and both selected approval record IDs, policy hashes, and install-trust hashes. Candidate-derived attestations and prompt provenance are evidence only and must exactly equal those external records. Seal creation performs the same semantic manifest, provenance, event, and coverage validation as standalone verification before invoking the signer. + +Every manifest and memory audit declares that the expert cache and KV cache are memory-resident and that there are no external cache or state paths. Missing, substituted, or additional file-backed cache/state declarations fail closed. + +The required hard-failure event components are `prompt.bytes`, `prompt.tokens`, `engram.row_ids`, `expert.ids`, `expert.weights`, `attn.source`, `attn.candidate_blocks`, `attn.candidates`, `logits.prefill`, `logits.decode`, and `decode.greedy_token`. `expert.ids` must declare `semantic_id_space: "original"`; cache slot IDs are rejected. Any graph tensor in the reserved `dsv41.trace.*` namespace with an unknown component, malformed suffix, or unexpected layer fails the exporter. + +Every bundle carries one strict runtime-discriminated accelerator attestation. A llama.cpp candidate uses the `strix-rocm` kind: the selected backend device must map through its PCI identity and Linux KFD topology to `gfx_target_version=110501` (`gfx1151`). A ds4 oracle uses the `apple-metal` kind: the selected Metal device records its registry ID, reported architecture, unified-memory property, and recommended working-set size. Missing kinds, unknown kinds, cross-kind fields, duplicate JSON keys, and mixed evidence fail closed. Cross-runtime comparison does not require the two physical accelerators or PCI identities to match; each runtime proves its own execution environment, while the model, prompt, inference semantics, and complete output artifacts remain exact comparison inputs. + +Internal tensors use raw ggml dimension order, and every dimension must be positive. The validator requires Engram rows as i32 `[24, token_count]`, original expert IDs as i32 `[6, token_count]`, router weights as f32 `[6, token_count]`, layer-0/1 raw attention-source rows as i32 `[128, token_count]`, compressed attention-source IDs as nonempty rank-2 i32 with width at most 512, layer-20 candidate blocks as nonempty rank-2 i32 with width at most 2048, propagated candidates as nonempty rank-2 i32 with width at most 512, and complete f32 logits as `[129280]`. Raw attention rows use physical ring IDs `0..127`, visible current-ubatch IDs `128..128+token_index`, and unavailable sentinel `128+token_count`; layers 0 and 1 must be byte-identical for each execution step. Original expert IDs must be within `0..383`. + +Validate or compare bundles: + +```sh +python3 tools/deepseek-v41-trace/trace_format.py validate TRACE \ + --signer-principal PRINCIPAL --lane strix-llama-candidate-v1 \ + --execution-challenge "$CHALLENGE" --run-id "$RUN_ID" \ + --candidate-exporter-policy-id "$CANDIDATE_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_SIGNATURE" \ + --approval-principal "$APPROVAL_PRINCIPAL" +python3 tools/deepseek-v41-trace/trace_format.py compare DS4_TRACE LLAMA_TRACE \ + --left-signer-principal DS4_PRINCIPAL --right-signer-principal LLAMA_PRINCIPAL \ + --execution-challenge "$CHALLENGE" \ + --left-run-id "$DS4_RUN_ID" --right-run-id "$LLAMA_RUN_ID" \ + --right-candidate-exporter-policy-id "$CANDIDATE_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_SIGNATURE" \ + --approval-principal "$APPROVAL_PRINCIPAL" \ + --report report.json +``` + +The first mismatch is reported by phase, decode step, exact token, layer, component, byte offset, flat element index, and per-token component element index. All required components use exact byte comparison. There is no tolerance mode. A ds4 bundle is invalid unless it reports revision `bd66c402070042bf0a79ad6ece8242de4c93680c`. + +## Corpus matrix + +Use only the target model and these repository files: + +```text +tests/corpus/correctness-prose.txt +tests/corpus/correctness-code.txt +tests/corpus/correctness-structured.txt +tests/corpus/correctness-numeric.txt +``` + +Their SHA-256 values are fixed in `trace_format.py`; the matrix refuses modified corpus bytes. The published GGUF must have SHA-256 `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`. + +Start at context 32768. `llama-deepseek-v41-prompt-builder` loads only the GGUF vocabulary, repeats each repository corpus deterministically, truncates the token sequence to `context - decode_steps`, detokenizes it, and requires exact token round-trip before saving the prompt. `run_matrix.py` creates each prompt once before either runtime executes and reuses its exact bytes for every chunk-boundary case. Run prefill plus at least eight greedy decode steps in one reused context. + +The initial correctness matrix uses the final admitted physical ubatch of 32. The worst-case routed expert union is `min(384, 6*32) = 192` experts per layer. The published GGUF metadata yields 398131200 bytes per cross-layer expert slot, so the required cache is exactly 76441190400 bytes, or 72900 MiB. The launchers reject other ubatch, slot, or cache-byte values. Host-memory admission still accounts the cache, staging, model, graph, state, outputs, current host use, and safety margin together before allocation. + +Run the oracle matrix only from the final integration commit that contains the canonical watchdog, memory admission, and this correctness harness. Supply its exact revision, its immutable merge-base, and a newly computed base-to-head binary diff SHA-256 to `run_matrix.py`; do not reuse the standalone correctness PR head or diff identity. + +ROCm on `gfx1151` is the primary acceptance backend: + +```sh +HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ + cmake -S . -B build-dsv41-trace-rocm \ + -DBUILD_SHARED_LIBS=ON \ + -DLLAMA_BUILD_TESTS=ON \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_APP=OFF \ + -DLLAMA_BUILD_UI=OFF \ + -DLLAMA_USE_PREBUILT_UI=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DFETCHCONTENT_FULLY_DISCONNECTED=ON \ + -DGGML_HIP=ON \ + -DGPU_TARGETS=gfx1151 \ + -DGGML_NATIVE=ON \ + -DCMAKE_BUILD_TYPE=Release +cmake --build build-dsv41-trace-rocm --config Release -j "$(nproc)" --target \ + llama-deepseek-v41-trace \ + llama-deepseek-v41-prompt-builder \ + test-deepseek41-trace-manifest \ + test-deepseek41-trace-injected \ + test-backend-ops \ + test-deepseek41-schema \ + test-deepseek41-engram \ + test-deepseek41-expert \ + test-deepseek41-memory \ + test-deepseek41-runtime \ + test-deepseek41-trace-host + +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o MUL_MAT_ID +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o MUL_MAT +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o GET_ROWS +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o SET_ROWS +build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o CPY +``` + +Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly, invokes the exact exporter for a pre-allocation device attestation, and rejects the run unless the backend PCI identity maps to exactly one KFD node reporting `gfx1151`. The native exporter repeats the query before model allocation and verifies that the loaded model still uses the same device. + +Static repository builds skip this shared-library trace component instead of failing configuration. + +Set `HIP_LAUNCH_BLOCKING=1` on the canonical watchdog command that owns the complete Strix matrix process group. The llama.cpp wrapper fails closed if this variable is absent or different, and every embedded Strix memory, swap, and watchdog audit records it. This Linux/ROCm setting is not an Apple Metal oracle requirement. + +## Strix candidate execution gate + +`run_llama.py` refuses model execution when swap is enabled, the canonical watchdog lease, heartbeat, or JSONL audit is missing or stale, another unrelated matching model workload is active, or any model/prompt/trace path fails the storage gate. + +The approved watchdog revision is exactly `778db6f50eae04e6c232c69b9575bdbd0747962b`, with `scripts/strix_memory_watchdog.py` SHA-256 `d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f`. That revision is an ancestor of this stack, so production validates the script in the exact candidate tree instead of copying it from another revision. Both Python validators and the native exporter reject every other revision or script hash. + +The approved watchdog must own the complete matrix process group and expose its canonical validation and process-group lease-guard APIs. The wrappers verify its pinned script identity, Python executable and argv position, PID and Linux start time, exact command bytes, 116/118/120 GiB thresholds, `/proc` source, watchdog/guardian/matrix topology, current process group, child command hash, atomic lease/heartbeat identities, heartbeat freshness and persistent-audit record hash, and the watchdog-held audit lock. The direct matrix payload starts the canonical process-group lease guard before inference. The wrappers repeat validation before and after each runtime. + +Use one empty directory on verified non-rotational NVMe for every Strix input and output. The Python launcher and both native tools resolve the nearest existing output parent through `/proc/self/mountinfo`, `/sys/dev/block`, and `/sys/class/block`. They require a resolvable local NVMe block device with `queue/rotational=0`; tmpfs, network filesystems, rotational disks, unknown devices, lexical or resolved `/mnt/bigspace` paths, and forbidden-root symlink escapes fail closed. Btrfs subvolume sources such as `/dev/nvme0n1p3[/home]` are resolved through the parent block device. `TMPDIR` is mandatory and must be an absolute literal pathname to an existing writable non-symlink directory on verified NVMe; there is no `/tmp` fallback. Relative paths and shell shorthand such as `~/tmp` are rejected because environment values are not shell-expanded for the launched process. These metadata commands do not execute the model: + +```sh +MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf +REPO=/home/papa/src/strix-llama-integration +RUN_ROOT=/home/papa/dsv41-correctness +TMPDIR=/home/papa/tmp/dsv41 +export TMPDIR + +test "$(stat -c %s "$MODEL")" = 365713686528 +test "$(realpath "$MODEL")" = /mnt/models/DeepSeek-V4.1-Flash-Q2.gguf +test "$(realpath -m "$REPO")" = /home/papa/src/strix-llama-integration +test "$(realpath -m "$RUN_ROOT")" = /home/papa/dsv41-correctness +test "$(realpath -m "$TMPDIR")" = /home/papa/tmp/dsv41 +mkdir -p "$RUN_ROOT" "$TMPDIR" +findmnt -no SOURCE,FSTYPE,TARGET -T "$MODEL" +findmnt -no SOURCE,FSTYPE,TARGET -T /home +lsblk -d -o NAME,ROTA,TYPE,SIZE,MODEL +df -B1 /mnt/models /home +sha256sum "$MODEL" +PYTHONPATH=gguf-py python3 -m gguf.scripts.gguf_dump "$MODEL" > "$RUN_ROOT/model-metadata.txt" +git -C /home/papa/src/ds4-v41 status --short +git -C /home/papa/src/ds4-v41 rev-parse HEAD +test "$(awk 'NR > 1 { count++ } END { print count + 0 }' /proc/swaps)" = 0 +``` + +The expected model digest is `1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`, both selected block devices report `ROTA=0`, and `/proc/swaps` has zero entries. The observed planning snapshot had 76366495744 bytes free on `/mnt/models` and 679635001344 bytes free under `/home`; recheck before every run. Keep the unchanged 365713686528-byte GGUF in place on `/mnt/models`. Do not copy the model or place builds, logs, traces, audit files, or temporary files there. Put all of those under `/home`, and never use `/mnt/bigspace`. + +The exporter is intentionally external to the canonical ds4 checkout. It must be built from the pinned revision and emit this trace format without changing the canonical checkout. The launcher requires its trusted SHA-256 and canonical executed path, and rejects a bundle unless the exporter reports the pinned revision and its build path and SHA-256 match the executed file. + +The llama.cpp exporter is built as `llama-deepseek-v41-trace`. It accepts the normal model, context, batch, ubatch, KV, Flash Attention, offload, and expert-cache arguments. `-bf` supplies the exact prompt bytes, `-n` is the number of greedy decode steps, and `-o` is the trace directory. It also requires `DSV41_TRACE_MEMORY_AUDIT`, `DSV41_TRACE_SWAP_AUDIT`, and `DSV41_TRACE_WATCHDOG_AUDIT` so every run points to its safety evidence. The content-addressed memory audit binds the preflight accelerator and storage attestations; the manifest binds the independently repeated native accelerator attestation. + +Use `run_llama.py` on the validation host instead of calling the exporter directly. It verifies the detached executable approval policy before invoking the exporter, then applies the same zero-swap, watchdog, active-workload, exact-`gfx1151`, and proven-NVMe gates and embeds content-addressed preflight and postflight evidence in the trace. It requires the exact producer revision, immutable base revision, expected base-to-producer binary diff SHA-256, verifier repository path and revision, external executable approval, externally approved signer principal, and matching private signing key. The private key is never copied or logged. Its public half is derived with the fixed trusted `ssh-keygen` executable and must exactly match the source-controlled signer map before model execution. + +The launcher checks the approved candidate exporter path, hash, device/inode, size, modification time, and change time before any exporter invocation and after each protected operation. The read-only `--dsv41-attest-build ROCm0` command loads the selected backend and emits the embedded runtime profile and receipt before model execution; the launcher requires exact equality with the external approval and repeats the attestation after trace completion. The native exporter embeds the full 40-character producer revision independently of dynamically loaded build-info, resolves its actual executable path, and validates every loaded `llama`, `ggml`, and enabled backend project library against the build-generated component receipt and selected runtime profile. Component name, filename, canonical path, SHA-256, role, and exact revision-bearing identity must match, and the measured loaded set must equal the predeclared profile set both immediately before protected trace generation and after it completes. Both snapshots and the completed loader-monitor receipt are signed. macOS also monitors loader additions during the protected interval. Missing, duplicated, unclassified, outside-root, renamed inside-root injected, catalogued-but-not-profile, changed, or late-loaded project libraries fail closed. Linux enumerates loaded ELF objects rather than arbitrary memory mappings and accepts non-project ROCm dependencies only from a root-owned, non-writable `/opt/rocm` installation. Production launchers and the native exporter reject dynamic-loader and `GGML_BACKEND_PATH` overrides. + +Production installs no manifest-writing or runtime-path probe option. With `LLAMA_BUILD_TESTS`, CMake builds the non-installed `test-deepseek41-trace-manifest` harness from the same native writer implementation. Its input accepts only model, prompt, audit, expected-coverage, and event-count data. Runtime, accelerator, path, configuration, build, and environment evidence comes from measured local state, and CPU or Darwin output is explicitly test-only and cannot satisfy production candidate or signer requirements. The trace install component places the exact receipt libraries beside the tools and sets `@loader_path/../lib` on macOS or `$ORIGIN/../lib` on ELF so installed `--version` needs no loader override. The install test also hashes every installed component and requires exact equality with the receipt embedded after the original library link; install-time rewriting or re-signing fails. + +`run_matrix.py --llama-only` runs the approved prompt builder on the Linux candidate host, copies the four repository corpora byte-for-byte into the NVMe result directory, verifies their fixed hashes, and verifies the approved builder path/hash, loaded runtime closure, complete tokenizer policy, and both original and copied corpus identities before prompt construction. It rechecks the builder, loaded libraries, and corpus identities after execution and requires the generated prompt hash, byte count, tokenizer policy, context, and decode configuration to equal the signed external approval before writing provenance. Pass the detached approval policy and signature, selected candidate and prompt approval IDs, both executables, the producer revision/base/diff identity, and the verifier checkout. Its default context matrix is 32768. Pass later contexts only after the 32K target passes. The Apple oracle is captured separately with `run_ds4.py`; compare completed per-case bundles with `trace_format.py compare`. + +## Apple Metal oracle execution gate + +The external ds4 exporter is not present in the pinned canonical checkout. Production remains blocked because the source-controlled `APPROVED_DS4_EXPORTERS` map is empty. A run requires a detached, externally signed executable-approval v2 policy with a selected `ds4_exporters` record. The record binds the `ds4` runtime role, `antirez/ds4` repository, pinned producer revision, distinct verifier revision, canonical install root and executable path, trusted owner, exact executable SHA-256, runtime profile, and exact dependency receipt. `run_ds4.py` verifies that policy before any exporter execution, requires a canonical non-symlink install hierarchy that is trusted-owned and not writable by the execution identity or an untrusted group, and requires the exporter and every dependency to be regular non-writable one-link trusted-owned files. Linux launches the retained exporter descriptor through `/proc/self/fd`. macOS launches the canonical path only after proving the immutable root and retains all measured descriptors through the launch; it does not claim descriptor-selected dyld loading. Test policies are explicit and cannot satisfy production defaults. + +Linux production execution must use an administrator-provisioned dedicated service identity whose live supplementary-group list is empty. The service may select one trusted primary group for required device or file access, or use exact administrator-managed ACLs, but it must not add the execution identity to `render`, `video`, or other supplementary groups. A normal login session is not valid, and an unprivileged process cannot repair a nonempty list after launch. Python checks the live list before it opens helper descriptors, and the helper checks `getgroups(0, nullptr) == 0` before it sends `READY` or creates a namespace, then rechecks the inherited empty list after both user-namespace mapping boundaries and after its credential drop. The signed containment-helper receipt binds `zero-supplementary-groups-v1` and an exact empty group list. Administrators must run `llama-deepseek-v41-containment-helper --check-launcher-groups` as `ExecStartPre=` under the same `User=` and `Group=` as the matrix service; success prints `supplementary-groups=0`. `SupplementaryGroups=` can add groups and does not replace this runtime check. Any nonzero list or query failure exits 125 before target code. + +Every exporter invocation repeats the install-root, executable, containment-helper, and dependency identity checks immediately before and after execution. Attestation calls have a fixed 60-second timeout and trace generation has a fixed 24-hour timeout, both bound by the signed runner script revision. Linux production execution requires a single-threaded supervisor and the receipt-bound native helper; Python does not use `fork()` at the containment boundary. Python launches the helper in a new session with `posix_spawn()` while all catchable signals are blocked and reset to default, obtains the helper pidfd before protocol release, and never uses numeric PID or process-group signaling. The helper verifies its inherited signal state, binds its lifetime to the exact Python parent with `PR_SET_PDEATHSIG`, and reports readiness before creating any target process. After `PREPARE`, `clone3(CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD)` atomically creates a blocked namespace init and stable pidfd. The helper verifies the namespace-init parent binding, pins its `/proc` identity, writes one-entry UID and GID maps for the trusted init identity, creates a private session, makes PID 1 non-dumpable, makes mount propagation private, accepts only `EINVAL` when a locked inherited `/proc` cannot be detached, mounts a new `nosuid,nodev,noexec` procfs over `/proc`, and verifies `/proc/self` resolves to namespace PID 1. A separate bounded binary diagnostic pipe reports setup stage and errno without sharing protocol tokens or exposing host data. The helper sends the namespace pidfd to Python only after that setup is acknowledged, and target execution remains blocked until Python verifies both stable identities and sends `EXEC`. PID 1 then uses `clone3(CLONE_NEWUSER | CLONE_PIDFD)` to create a target bootstrap in a nested user namespace, maps only namespace UID and GID 65534 to the trusted parent namespace identity, and requires the target to create a separate session. Before `execve()`, the bootstrap verifies that the inherited supplementary-group list is still empty, locks securebits against root or set-ID capability regeneration, changes all real, effective, and saved credentials to 65534, drops every bounding, effective, permitted, inheritable, and ambient capability, sets `PR_SET_NO_NEW_PRIVS`, and verifies each state. It then installs an inherited seccomp user-notification filter for ptrace, process-vm access, PID 1 or process-group signaling, lifecycle `prctl`, credential or capability changes, session or process-group changes, and namespace entry. PID 1 owns the only listener, validates the bootstrap denial probes, and treats any later notification from any target descendant as fatal to the complete PID namespace. PID 1 re-verifies its parent-death signal, non-dumpable state, isolated session, and empty supplementary-group list before it confirms target isolation to the helper. The namespace init executes the target only after this handshake, then kills and reaps every remaining namespace member. If Python dies, the kernel kills the helper; helper death kills namespace PID 1; namespace PID 1 death kills the complete target namespace. If the dedicated zero-group service contract, unprivileged user, PID, or mount namespaces, nested target-user mapping, clone3, pidfds, credential or capability drops, securebits, no-new-privileges, seccomp notification, parent-death binding, private sessions, non-dumpable PID 1, private procfs, protocol identity, or helper receipt verification are unavailable, execution fails before target code. The helper sends `COMPLETE` only after the namespace is empty, reaped, and its pidfd is closed. If cleanup cannot prove that state, Python retains the stable containment authority and lock and poisons the supervisor against reuse. Other POSIX platforms fail closed because a process group is not a containment boundary. The Darwin process-group path is an explicit test-only fixture and is never production-valid. Windows creates the process suspended, records assignment and resume state, assigns it to a kill-on-close Job Object before resume, and directly terminates and reaps the exact suspended child if assignment fails. The `subprocess.Handle` wrapper retains sole ownership of the process handle and closes it exactly once; raw `CloseHandle` is not used for that wrapper. A structured completion result gates every post-attestation, and any containment acquisition, protocol, termination, assignment, reaping, empty-set proof, helper completion, or executable, runtime, helper, stream, pidfd, Job, process, or thread descriptor teardown failure keeps that result false. Captured output remains bytes until process-tree cleanup and all lower identity checks finish; DS4 strict UTF-8 decoding is then captured as the operation primary before post-build attestation. A non-attestation invocation always runs a post-invocation build attestation before its result or exception is honored when containment completion is proven; if execution, cleanup, postchecks, decoding, or post-attestation fail together, the launcher preserves the primary error and every typed secondary failure. The exporter must answer `--dsv41-attest-build` without loading the model and report its exact path, SHA-256, runtime profile, dependency receipt digest, and pre/post loaded-library closure. The launcher measures that evidence before and after device attestation, trace generation, and postflight device attestation. The signed authorization, oracle evidence, runner audits, and Seal v1 manifest bind the DS4 approval ID and hash, install-trust hash, canonical executable identity, runtime profile and receipt, loaded-library closure, and producer/verifier revisions. Missing or mismatched policy, mutable or aliased paths, hard links, component substitution, and changed build evidence fail closed. `run_ds4.py` also requires macOS arm64, at least 128 GiB of measured host memory, zero swap, no unrelated matching workload, an exact selected Metal device query, and an existing writable non-symlink `TMPDIR`. The model, prompt, output, harness repository, ds4 checkout, temporary directory, Python executable, runner script, and exporter must resolve through `df -P` to a volume that `diskutil info -plist` proves is internal solid-state storage backed by NVMe or Apple Fabric. SATA, network, virtual, disk-image, external/non-internal, non-solid-state, and incomplete device identities fail closed, as do lexical or resolved forbidden paths. + +The ds4 memory audit binds the exact Metal accelerator, host model/OS/memory identity, and every storage record. The runner audit binds the Python runner process, UID, executable/script paths and hashes, exporter path/hash, external approval and install-trust hashes, runtime build/profile/receipt identity, producer and verifier revisions, pinned checkout path/revision, and exact command hash. The runner script must be inside the attested harness repository. The preflight and postflight accelerator and host identities must remain unchanged. These Apple audits replace Linux KFD, `/proc`, HIP, and Strix watchdog claims; the oracle must never fabricate those fields. + +After the exporter and its complete runtime receipt are independently reviewed on an authorized 128 GiB or larger Apple oracle host, add the externally reviewed record to a signed executable-approval v2 policy and select it with `--ds4-exporter-policy-id`. Do not add test policy material to the production maps. A caller-provided digest or self-reported build record alone is not sufficient oracle provenance. The exporter must answer `--dsv41-attest-build` and `--dsv41-attest-device Metal0` without loading the model. The device query emits the strict `apple-metal` attestation. Its trace command interface is: + +The unpublished `ds4gguf` documentation revision `e13893ffcb33e90c8852929303e188102df7a8f5` is provenance only. It is not an executable dependency, exporter approval, or fixture source. Executable tests and fixtures stay in this `strix-llama.cpp` stack. + +```text +--model PATH --prompt-file PATH --output PATH --context N --decode-steps N --prefill-chunk 32 --device Metal0 +``` + +It must emit a complete valid `dsv41-trace` bundle, report ds4 revision `bd66c402070042bf0a79ad6ece8242de4c93680c`, put its own executable SHA-256 in `manifest.json`, and report the same selected Metal device before and after execution. `run_ds4.py` embeds the platform-native memory, swap, runner, accelerator, host, storage, and exact-path evidence. + +Capture the first llama.cpp matrix under the watchdog: + +```sh +REPO=/home/papa/src/strix-llama-integration +MODEL=/mnt/models/DeepSeek-V4.1-Flash-Q2.gguf +RUN_ROOT=/home/papa/dsv41-correctness +CASE_ROOT="$RUN_ROOT/c32768" +CANDIDATE_REV="$(git -C "$REPO" rev-parse HEAD)" +BASE_REV= +DIFF_SHA256="$(git -C "$REPO" diff --binary --no-ext-diff "$BASE_REV" "$CANDIDATE_REV" -- | sha256sum | awk '{print $1}')" +CHALLENGE=<64-lowercase-hex-execution-challenge> +AUTH_ISSUED= +AUTH_EXPIRES= +APPROVAL_POLICY= +APPROVAL_SIGNATURE= +APPROVAL_PRINCIPAL= +CANDIDATE_EXPORTER_POLICY_ID= +PROMPT_BUILDER_POLICY_ID= + +mkdir -p "$CASE_ROOT/watchdog" +cd "$REPO" +HIP_LAUNCH_BLOCKING=1 python3 scripts/strix_memory_watchdog.py \ + --procfs-root /proc \ + --soft-gib 116 \ + --emergency-gib 118 \ + --grace-seconds 30 \ + --sample-interval-seconds 1 \ + --lease-path "$CASE_ROOT/watchdog/lease.json" \ + --heartbeat-path "$CASE_ROOT/watchdog/heartbeat.json" \ + --audit-path "$CASE_ROOT/watchdog/audit.jsonl" \ + --heartbeat-max-age-seconds 5 \ + -- \ + python3 tools/deepseek-v41-trace/run_matrix.py \ + --repo "$REPO" \ + --model "$MODEL" \ + --output "$CASE_ROOT/matrix" \ + --llama-runner "$REPO/tools/deepseek-v41-trace/run_llama.py" \ + --llama-exporter "$REPO/build-dsv41-trace-rocm/bin/llama-deepseek-v41-trace" \ + --llama-prompt-builder "$REPO/build-dsv41-trace-rocm/bin/llama-deepseek-v41-prompt-builder" \ + --candidate-revision "$CANDIDATE_REV" \ + --base-revision "$BASE_REV" \ + --candidate-diff-sha256 "$DIFF_SHA256" \ + --candidate-exporter-policy-id "$CANDIDATE_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_SIGNATURE" \ + --approval-principal "$APPROVAL_PRINCIPAL" \ + --signer-principal "$LLAMA_SIGNER_PRINCIPAL" \ + --signing-key "$LLAMA_SIGNING_KEY" \ + --execution-challenge "$CHALLENGE" \ + --run-id-prefix "strix-llama-c32768" \ + --authorization-issued-unix "$AUTH_ISSUED" \ + --authorization-expires-unix "$AUTH_EXPIRES" \ + --llama-only \ + --contexts 32768 \ + --ubatches 32 \ + --batch 2048 \ + --device ROCm0 \ + --expert-cache-slots 192 \ + --expert-cache-mib 72900 +``` + +Use a new empty output and watchdog directory for each later context. Repeat the same command after setting `CASE_ROOT` and changing `--contexts`: + +```sh +CASE_ROOT="$RUN_ROOT/c65536" # then use --contexts 65536 +CASE_ROOT="$RUN_ROOT/c98304" # then use --contexts 98304 +CASE_ROOT="$RUN_ROOT/c131072" # then use --contexts 131072 +``` + +On the separately authorized Apple oracle, place the unchanged GGUF, exact prompt, prompt-provenance record, harness checkout, pinned ds4 checkout, exporter, trace output, and `TMPDIR` on internal solid-state storage. Then run one case at a time: + +```sh +export TMPDIR=/Users/oracle/dsv41/tmp +python3 tools/deepseek-v41-trace/run_ds4.py \ + --repo /Users/oracle/src/strix-llama.cpp \ + --checkout /Users/oracle/src/ds4-v41 \ + --exporter /opt/dsv41/ds4/bin/dsv41-trace-exporter \ + --exporter-sha256 \ + --model /Users/oracle/models/DeepSeek-V4.1-Flash-Q2.gguf \ + --prompt /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt \ + --prompt-provenance /Users/oracle/dsv41/inputs/correctness-prose-c32768.txt.provenance.json \ + --output /Users/oracle/dsv41/traces/correctness-prose-c32768-ub32 \ + --approval-policy "$APPROVAL_POLICY" \ + --approval-signature "$APPROVAL_POLICY_SIGNATURE" \ + --approval-principal "$APPROVAL_APPROVER_PRINCIPAL" \ + --ds4-exporter-policy-id "$DS4_EXPORTER_POLICY_ID" \ + --prompt-builder-policy-id "$PROMPT_BUILDER_POLICY_ID" \ + --corpus-name correctness-prose.txt \ + --corpus-sha256 2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2 \ + --context 32768 \ + --decode-steps 8 \ + --prefill-chunk 32 \ + --device Metal0 \ + --signer-principal "$DS4_SIGNER_PRINCIPAL" \ + --signing-key "$DS4_SIGNING_KEY" \ + --execution-challenge "$CHALLENGE" \ + --run-id "apple-ds4-prose-c32768-ub32" \ + --authorization-issued-unix "$AUTH_ISSUED" \ + --authorization-expires-unix "$AUTH_EXPIRES" +``` + +Compare the completed bundle with the matching Strix bundle using `trace_format.py compare`. Repeat for all four corpora before expanding the context matrix. + +## Strix bring-up evidence lane + +The 128 GiB or larger Apple Metal run remains an external verification gate for the pinned ds4 cross-runtime oracle. A Strix bring-up can complete first, but it must report `BRINGUP PASS`, never `TARGET PASS`. It does not replace complete byte-identical ds4 logits. + +The pinned `antirez/ds4@bd66c402070042bf0a79ad6ece8242de4c93680c` evidence anchors are: + +| Evidence | SHA-256 | Limitation | +|---|---|---| +| `tests/test-vectors/README.md` | `0e59b2f2832bed8af0a91e6ff20962debf964cd2d1d141c086e52cfcc995a1c3` | Official-vector provenance and limitations | +| `tests/test-vectors/flash-0731/manifest.json` | `ebf237a5660a6851fb8085e77f532901a9d758208b25d7ed5af0b7af4b28f91b` | Official API provenance | +| `tests/test-vectors/flash-0731/official.vec` | `77ae699889bfaf1348768dcbe7ea2c72279ae86abb10470d3e1b08cd1fd82a83` | Official selected-token/top-logprob slice, not full logits | +| `tests/test-vectors/flash-0731/local-golden.vec` | `23d942ff3b9bb2a3f82927d11aa3ed1461e1f302071e788d0d95a5c165e47d3b` | Local tolerant top-64 drift anchor, not exact | +| `tests/test_engram.c` | `198a561d981f62518a9d28035480a7e220b99c156cde6d248b8baabd684cc74b` | Model-free Engram oracle | +| `ds4_engram.c` | `2b6ca468510ebf45ee298a905525bc7234dacad9a384bf2011eba19ba2c0bdf7` | Engram implementation under test | +| `ds4_engram.h` | `f84a264e0fe199d23a6f0c56fbbd19e222adc7009eed13c185af6403f68f7f5c` | Engram schema | +| `tests/test_deepseek41_metal.c` | `9197c2f9d65b380ce25be5334991e4bfaf40e82e6708e64f2b9329552412d28c` | Synthetic exact candidate/top-k oracle; source anchor only on Strix | +| `tests/test_deepseek41_graph.c` | `6dc786f831c93ae7f5aa56e7518f67657125f7c0fd35cead3c646eff3d3f9e09` | Synthetic routing and graph oracle | +| `tests/test_deepseek41_prefill.c` | `452774b9332d393822d84288eeb2d71ca1fb25f1ba30d20a49160d1787de734f` | Prefill boundary fixture | +| `tests/test_deepseek41_manifest.py` | `2d7aa1fc93805d9c97839c5de9eccc856628f13e6913c3bbc3f0c99b0827aeae` | Model manifest/schema oracle | +| `tests/test_deepseek41_conversion.py` | `a40b83062a9b91338773addd77fe62650296f4de607057cb4fd1329287f347b5c` | Conversion/schema fixture | +| `tests/test_deepseek41_gguf.c` | `8f41e049d5ec179c38a1a00ac61712db0306902f0ef74bcdb989d8993316ff35` | GGUF schema oracle | +| `gguf-tools/deepseek41_metadata.py` | `39300bbd504165b97de017edd72563377f50b8a5511ca7478252a86f31a0009b` | Conversion metadata source | +| `ds4.c` | `1776dbfed177ea14f3ce6cac1d8d0b1c1b44dfff2c2663769a9a5634aeec34e7` | Runtime schema source | + +Verify these files from the pinned checkout before using their results. Do not copy an unpublished exporter or depend on a private ds4 remote. + +```sh +python3 tools/deepseek-v41-trace/verify_ds4_anchors.py \ + --checkout /home/papa/src/ds4-v41 +``` + +`run_matrix.py --llama-only` captures all four repository corpora without claiming cross-runtime success. Run the final integration build twice with separate output directories, then run an equivalently instrumented immutable base build once. Keep every run under its own canonical watchdog invocation and use the exact ubatch/cache/ROCm arguments above. + +For each case (`correctness-prose-c32768-ub32`, `correctness-code-c32768-ub32`, `correctness-structured-c32768-ub32`, and `correctness-numeric-c32768-ub32`), require both comparisons: + +```sh +python3 tools/deepseek-v41-trace/trace_format.py compare-local self-consistency \ + "$RUN_A/llama/$CASE" "$RUN_B/llama/$CASE" \ + --left-signer-principal "$SIGNER_PRINCIPAL" \ + --right-signer-principal "$SIGNER_PRINCIPAL" \ + --execution-challenge "$CHALLENGE" \ + --left-run-id "$RUN_A_ID" --right-run-id "$RUN_B_ID" \ + --report "$REPORTS/$CASE-self.json" + +python3 tools/deepseek-v41-trace/trace_format.py compare-local base-regression \ + "$BASE_RUN/llama/$CASE" "$RUN_A/llama/$CASE" \ + --left-signer-principal "$SIGNER_PRINCIPAL" \ + --right-signer-principal "$SIGNER_PRINCIPAL" \ + --execution-challenge "$CHALLENGE" \ + --left-run-id "$BASE_RUN_ID" --right-run-id "$RUN_A_ID" \ + --report "$REPORTS/$CASE-base.json" +``` + +Both commands compare every exact trace component, including complete prefill/decode logits, tokens, Engram rows, original expert IDs and weights, raw/compressed attention sources, and candidate propagation. `base-regression` also requires the base trace revision to equal the integrated trace's attested oracle revision. The report includes `cross_runtime_status: "INCOMPLETE"` even when it returns `BRINGUP PASS`. + +The pinned ds4 evidence commands are: + +```sh +git -C /home/papa/src/ds4-v41 diff --quiet +git -C /home/papa/src/ds4-v41 diff --cached --quiet +test "$(git -C /home/papa/src/ds4-v41 rev-parse HEAD)" = bd66c402070042bf0a79ad6ece8242de4c93680c +/home/papa/src/ds4-v41/tests/test_engram +DS4_TEST_MODEL="$MODEL" \ +DS4_TEST_VECTOR_FILE=/home/papa/src/ds4-v41/tests/test-vectors/flash-0731/official.vec \ + /home/papa/src/ds4-v41/ds4_test --logprob-vectors +``` + +Capture the exact commands, executable hashes, stdout/stderr hashes, exit status, and watchdog artifacts. The official-vector result and pinned fixtures are supporting bring-up evidence only. The first `TARGET PASS` still requires the external pinned ds4 exporter and the full cross-runtime trace comparison. diff --git a/tools/deepseek-v41-trace/generate-containment-helper-receipt.py b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py new file mode 100644 index 000000000000..3731c7931dc6 --- /dev/null +++ b/tools/deepseek-v41-trace/generate-containment-helper-receipt.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--helper", required=True, type=Path) + parser.add_argument("--revision", required=True) + args = parser.parse_args() + if re.fullmatch(r"[0-9a-f]{40}", args.revision) is None: + parser.error("revision must be an exact full Git revision") + helper = args.helper.resolve(strict=True) + receipt = { + "format": "dsv41-containment-helper", + "version": 2, + "revision": args.revision, + "filename": helper.name, + "sha256": sha256_file(helper), + "launcher_policy": "zero-supplementary-groups-v1", + "supplementary_groups": [], + } + content = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_suffix(args.output.suffix + ".tmp") + temporary.write_bytes(content) + os.replace(temporary, args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/generate-runtime-receipt.py b/tools/deepseek-v41-trace/generate-runtime-receipt.py new file mode 100644 index 000000000000..9beb3024fab3 --- /dev/null +++ b/tools/deepseek-v41-trace/generate-runtime-receipt.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path + + +def quote(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--json-output", required=True, type=Path) + parser.add_argument("--revision", required=True) + parser.add_argument("--profile", required=True, choices=("co-located", "sibling-lib")) + parser.add_argument("--entry", action="append", nargs=3, default=[], metavar=("COMPONENT", "PATH", "REVISION")) + args = parser.parse_args() + + if re.fullmatch(r"[0-9a-f]{40}", args.revision) is None: + parser.error("revision must be an exact full Git revision") + if not args.entry: + parser.error("at least one receipt entry is required") + + entries = [] + components = set() + filenames = set() + digests = set() + for component, path_text, revision in args.entry: + path = Path(path_text).resolve(strict=True) + digest = sha256_file(path) + filename = path.name + if re.fullmatch(r"[a-z0-9-]+", component) is None: + parser.error(f"invalid component: {component}") + if re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None: + parser.error(f"invalid filename: {filename}") + if component in components: + parser.error(f"duplicate component: {component}") + if filename in filenames: + parser.error(f"duplicate filename: {filename}") + if digest in digests: + parser.error(f"duplicate SHA-256: {digest}") + if revision != "-" and revision != args.revision: + parser.error(f"invalid revision for {component}") + components.add(component) + filenames.add(filename) + digests.add(digest) + entries.append((component, filename, digest, "" if revision == "-" else revision)) + + entries.sort() + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": args.revision, + "profile": args.profile, + "components": [ + { + "component": component, + "filename": filename, + "sha256": digest, + "revision": revision or None, + } + for component, filename, digest, revision in entries + ], + } + receipt_sha256 = hashlib.sha256( + json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode("ascii") + ).hexdigest() + lines = [ + "#pragma once", + "", + "#include ", + "#include ", + "", + "namespace dsv41_runtime_receipt {", + "", + "struct entry {", + " const char * component;", + " const char * filename;", + " const char * sha256;", + " const char * revision;", + "};", + "", + f"inline constexpr const char * profile = {quote(args.profile)};", + f"inline constexpr const char * sha256 = {quote(receipt_sha256)};", + f"inline constexpr std::array entries = {{{{", + ] + for component, filename, digest, revision in entries: + lines.append( + " {" + + ", ".join((quote(component), quote(filename), quote(digest), quote(revision))) + + "}," + ) + lines.extend([ + "}};", + f"inline constexpr std::array components = {{{{", + ]) + for component, _, _, _ in entries: + lines.append(f" {quote(component)},") + lines.extend([ + "}};", + "", + "}", + "", + ]) + content = "\n".join(lines).encode("ascii") + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_suffix(args.output.suffix + ".tmp") + temporary.write_bytes(content) + os.replace(temporary, args.output) + receipt_content = ( + json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("ascii") + args.json_output.parent.mkdir(parents=True, exist_ok=True) + receipt_temporary = args.json_output.with_suffix(args.json_output.suffix + ".tmp") + receipt_temporary.write_bytes(receipt_content) + os.replace(receipt_temporary, args.json_output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/host-attestation.h b/tools/deepseek-v41-trace/host-attestation.h new file mode 100644 index 000000000000..1f621237d048 --- /dev/null +++ b/tools/deepseek-v41-trace/host-attestation.h @@ -0,0 +1,371 @@ +#pragma once + +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#endif + +namespace dsv41 { + +namespace fs = std::filesystem; + +struct storage_attestation { + fs::path resolved_path; + fs::path existing_path; + std::string mount_point; + std::string filesystem_type; + std::string mount_source; + std::string device_number; + fs::path block_device_path; + std::string nvme_device; +}; + +struct accelerator_attestation { + std::string backend_device; + std::string backend_description; + std::string pci_device_id; + std::string kfd_node; + uint64_t gpu_id; + uint64_t gfx_target_version; + std::string architecture; +}; + +static inline std::string read_text(const fs::path & path) { + std::ifstream input(path); + if (!input) { + throw std::runtime_error("cannot read " + path.string()); + } + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +static inline std::string trim(std::string value) { + while (!value.empty() && std::isspace(static_cast(value.back()))) { + value.pop_back(); + } + size_t first = 0; + while (first < value.size() && std::isspace(static_cast(value[first]))) { + ++first; + } + return value.substr(first); +} + +static inline std::string decode_mount_field(const std::string & value) { + std::string result; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '\\' && i + 3 < value.size() && + value[i + 1] >= '0' && value[i + 1] <= '7' && + value[i + 2] >= '0' && value[i + 2] <= '7' && + value[i + 3] >= '0' && value[i + 3] <= '7') { + const int byte = (value[i + 1] - '0')*64 + (value[i + 2] - '0')*8 + value[i + 3] - '0'; + result.push_back(static_cast(byte)); + i += 3; + } else { + result.push_back(value[i]); + } + } + return result; +} + +static inline bool path_is_within(const fs::path & path, const fs::path & root) { + if (path == root) { + return true; + } + const fs::path relative = path.lexically_relative(root); + return !relative.empty() && *relative.begin() != ".."; +} + +static inline fs::path existing_ancestor(const fs::path & path) { + fs::path current = path; + while (!current.empty() && !fs::exists(current)) { + const fs::path parent = current.parent_path(); + if (parent == current) { + break; + } + current = parent; + } + if (current.empty() || !fs::exists(current)) { + throw std::runtime_error("path has no existing parent: " + path.string()); + } + return fs::canonical(current); +} + +static inline void reject_symlink_components(const fs::path & path, const char * label) { + fs::path current; + for (const fs::path & part : fs::absolute(path)) { + current /= part; + std::error_code error; + const fs::file_status status = fs::symlink_status(current, error); + if (!error && fs::is_symlink(status)) { + throw std::runtime_error(std::string(label) + " must not be a symlink or contain symlink components"); + } + if (error == std::errc::no_such_file_or_directory) { + break; + } + if (error) { + throw std::runtime_error( + std::string(label) + " symlink status cannot be read: " + error.message()); + } + } +} + +static inline void require_usable_directory(const fs::path & path, const char * label) { + reject_symlink_components(path, label); + if (!fs::is_directory(path)) { + throw std::runtime_error(std::string(label) + " must be an existing directory"); + } +#if defined(_WIN32) + if (_access(path.string().c_str(), 6) != 0) { +#else + if (access(path.string().c_str(), W_OK | X_OK) != 0) { +#endif + throw std::runtime_error(std::string(label) + " must be writable and searchable"); + } +} + +static inline storage_attestation require_nvme_path( + const fs::path & path, + const char * label, + const fs::path & mountinfo_path = "/proc/self/mountinfo", + const fs::path & sys_dev_block_root = "/sys/dev/block", + const fs::path & sys_class_block_root = "/sys/class/block", + const fs::path & forbidden_root = "/mnt/bigspace") { + const fs::path absolute = fs::absolute(path).lexically_normal(); + const fs::path forbidden = fs::absolute(forbidden_root).lexically_normal(); + if (path_is_within(absolute, forbidden)) { + throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); + } + reject_symlink_components(path, label); + const fs::path resolved = fs::weakly_canonical(absolute); + const fs::path existing = existing_ancestor(resolved); + if (path_is_within(resolved, forbidden)) { + throw std::runtime_error(std::string(label) + " must not use /mnt/bigspace"); + } + + struct mount_record { + fs::path mount_point; + std::string filesystem_type; + std::string source; + std::string device_number; + }; + std::vector mounts; + std::istringstream mountinfo(read_text(mountinfo_path)); + std::string line; + while (std::getline(mountinfo, line)) { + std::istringstream fields_stream(line); + std::vector fields; + std::string field; + while (fields_stream >> field) { + fields.push_back(field); + } + const auto separator = std::find(fields.begin(), fields.end(), "-"); + if (fields.size() < 7 || separator == fields.end() || separator + 2 >= fields.end()) { + throw std::runtime_error("mountinfo contains an invalid record"); + } + mounts.push_back({ + decode_mount_field(fields[4]), + *(separator + 1), + decode_mount_field(*(separator + 2)), + fields[2], + }); + } + + const mount_record * selected = nullptr; + for (const mount_record & mount : mounts) { + if (path_is_within(existing, mount.mount_point) && + (selected == nullptr || mount.mount_point.string().size() > selected->mount_point.string().size())) { + selected = &mount; + } + } + if (selected == nullptr) { + throw std::runtime_error(std::string(label) + " mount cannot be resolved: " + resolved.string()); + } + + std::string device_number = selected->device_number; + if (device_number.substr(0, device_number.find(':')) == "0") { + const std::string source = selected->source.substr(0, selected->source.find('[')); + const fs::path source_path = source; + const fs::path source_dev = sys_class_block_root / source_path.filename() / "dev"; + if (source.rfind("/dev/", 0) != 0 || !fs::is_regular_file(source_dev)) { + throw std::runtime_error( + std::string(label) + " is not backed by a local block device: " + resolved.string()); + } + device_number = trim(read_text(source_dev)); + static const std::regex device_number_pattern("^[0-9]+:[0-9]+$"); + if (!std::regex_match(device_number, device_number_pattern)) { + throw std::runtime_error(std::string(label) + " backing device identity is invalid"); + } + } + + const fs::path device_link = sys_dev_block_root / device_number; + if (!fs::exists(device_link)) { + throw std::runtime_error( + std::string(label) + " is not backed by a resolvable block device: " + resolved.string()); + } + const fs::path block_device = fs::canonical(device_link); + fs::path rotational_path; + for (fs::path current = block_device; !current.empty(); current = current.parent_path()) { + const fs::path candidate = current / "queue" / "rotational"; + if (fs::is_regular_file(candidate)) { + rotational_path = candidate; + break; + } + if (current == current.parent_path()) { + break; + } + } + if (rotational_path.empty()) { + throw std::runtime_error( + std::string(label) + " block device rotational state cannot be resolved: " + block_device.string()); + } + if (trim(read_text(rotational_path)) != "0") { + throw std::runtime_error(std::string(label) + " must use non-rotational storage: " + resolved.string()); + } + + static const std::regex nvme_pattern("^nvme[0-9]+(c[0-9]+)?n[0-9]+$"); + std::string nvme_device; + for (const fs::path & component : block_device) { + const std::string name = component.string(); + if (std::regex_match(name, nvme_pattern)) { + nvme_device = name; + } + } + if (nvme_device.empty()) { + throw std::runtime_error( + std::string(label) + " must use an NVMe block device: " + block_device.string()); + } + + return { + resolved, + existing, + selected->mount_point.string(), + selected->filesystem_type, + selected->source, + device_number, + block_device, + nvme_device, + }; +} + +static inline std::map read_kfd_properties(const fs::path & path) { + std::map result; + std::istringstream input(read_text(path)); + std::string key; + uint64_t value = 0; + while (input >> key >> value) { + result[key] = value; + std::string rest; + std::getline(input, rest); + } + return result; +} + +static inline std::string gfx_architecture(uint64_t version) { + const uint64_t major = version / 10000; + const uint64_t minor = (version / 100) % 100; + const uint64_t stepping = version % 100; + if (major == 0 || minor > 9 || stepping > 9) { + throw std::runtime_error("KFD gfx_target_version is invalid"); + } + return "gfx" + std::to_string(major) + std::to_string(minor) + std::to_string(stepping); +} + +static inline accelerator_attestation require_gfx1151_identity( + const std::string & backend_device, + const std::string & backend_description, + const std::string & pci_device_id, + const fs::path & kfd_nodes_root) { + if (backend_device != "ROCm0") { + throw std::runtime_error("selected execution device must be ROCm0"); + } + if (backend_description.empty()) { + throw std::runtime_error("selected ROCm device description is missing"); + } + + static const std::regex pci_pattern("^([0-9a-f]{4}):([0-9a-f]{2}):([0-9a-f]{2})\\.([0-7])$"); + std::smatch match; + if (!std::regex_match(pci_device_id, match, pci_pattern)) { + throw std::runtime_error("selected ROCm device PCI identity is missing or invalid"); + } + const uint64_t domain = std::stoull(match[1].str(), nullptr, 16); + const uint64_t bus = std::stoull(match[2].str(), nullptr, 16); + const uint64_t slot = std::stoull(match[3].str(), nullptr, 16); + const uint64_t function = std::stoull(match[4].str(), nullptr, 16); + const uint64_t location_id = (bus << 8) | (slot << 3) | function; + + std::vector matches; + if (!fs::is_directory(kfd_nodes_root)) { + throw std::runtime_error("KFD topology is unavailable"); + } + for (const fs::directory_entry & entry : fs::directory_iterator(kfd_nodes_root)) { + if (!entry.is_directory()) { + continue; + } + const fs::path properties_path = entry.path() / "properties"; + const fs::path gpu_id_path = entry.path() / "gpu_id"; + if (!fs::is_regular_file(properties_path) || !fs::is_regular_file(gpu_id_path)) { + continue; + } + const std::map properties = read_kfd_properties(properties_path); + const auto domain_value = properties.find("domain"); + const auto location_value = properties.find("location_id"); + const auto gfx_value = properties.find("gfx_target_version"); + if (domain_value == properties.end() || location_value == properties.end() || + gfx_value == properties.end() || domain_value->second != domain || + location_value->second != location_id) { + continue; + } + const uint64_t gpu_id = std::stoull(trim(read_text(gpu_id_path))); + if (gpu_id == 0) { + throw std::runtime_error("selected KFD topology node has no GPU identity"); + } + matches.push_back({ + backend_device, + backend_description, + pci_device_id, + entry.path().filename().string(), + gpu_id, + gfx_value->second, + gfx_architecture(gfx_value->second), + }); + } + if (matches.size() != 1) { + throw std::runtime_error("selected ROCm device does not map to exactly one KFD topology node"); + } + if (matches[0].architecture != "gfx1151" || matches[0].gfx_target_version != 110501) { + throw std::runtime_error( + "selected ROCm device architecture must be gfx1151, found " + matches[0].architecture); + } + return matches[0]; +} + +static inline accelerator_attestation require_gfx1151_device( + ggml_backend_dev_t device, + const fs::path & kfd_nodes_root = "/sys/class/kfd/kfd/topology/nodes") { + if (device == nullptr) { + throw std::runtime_error("required ROCm device is unavailable"); + } + ggml_backend_dev_props props = {}; + ggml_backend_dev_get_props(device, &props); + return require_gfx1151_identity( + props.name == nullptr ? "" : props.name, + props.description == nullptr ? "" : props.description, + props.device_id == nullptr ? "" : props.device_id, + kfd_nodes_root); +} + +} diff --git a/tools/deepseek-v41-trace/linux-containment-helper.cpp b/tools/deepseek-v41-trace/linux-containment-helper.cpp new file mode 100644 index 000000000000..8d4da109c13d --- /dev/null +++ b/tools/deepseek-v41-trace/linux-containment-helper.cpp @@ -0,0 +1,1481 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(SYS_clone3) +#define SYS_clone3 435 +#endif +#if !defined(SYS_pidfd_send_signal) +#define SYS_pidfd_send_signal 424 +#endif +#if !defined(SYS_pidfd_open) +#define SYS_pidfd_open 434 +#endif +#if !defined(SYS_seccomp) +#define SYS_seccomp 317 +#endif +#if !defined(SECCOMP_RET_KILL_PROCESS) +#define SECCOMP_RET_KILL_PROCESS SECCOMP_RET_KILL +#endif +#if !defined(PR_SET_PTRACER) +#define PR_SET_PTRACER 0x59616d61 +#endif + +extern char ** environ; +#endif + +#ifndef DSV41_BUILD_REVISION +#define DSV41_BUILD_REVISION "unknown" +#endif + +namespace { + +struct options { + int protocol_fd = -1; + int expected_parent = -1; + std::string exec_path; + std::vector keep_fds; + std::vector target_argv; +}; + +int parse_positive_int(const char * value, const char * label) { + char * end = nullptr; + errno = 0; + const long parsed = std::strtol(value, &end, 10); + if (errno != 0 || end == value || *end != '\0' || parsed <= 0 || + parsed > std::numeric_limits::max()) { + throw std::runtime_error(std::string("invalid ") + label); + } + return static_cast(parsed); +} + +options parse_options(int argc, char ** argv) { + options result; + int index = 1; + while (index < argc && std::strcmp(argv[index], "--") != 0) { + if (std::strcmp(argv[index], "--protocol-fd") == 0 && index + 1 < argc) { + result.protocol_fd = parse_positive_int(argv[index + 1], "protocol descriptor"); + index += 2; + } else if (std::strcmp(argv[index], "--expected-parent") == 0 && index + 1 < argc) { + result.expected_parent = parse_positive_int(argv[index + 1], "expected parent"); + index += 2; + } else if (std::strcmp(argv[index], "--exec-path") == 0 && index + 1 < argc) { + result.exec_path = argv[index + 1]; + index += 2; + } else if (std::strcmp(argv[index], "--keep-fd") == 0 && index + 1 < argc) { + result.keep_fds.push_back(parse_positive_int(argv[index + 1], "retained descriptor")); + index += 2; + } else { + throw std::runtime_error("unknown containment helper argument"); + } + } + if (index >= argc || std::strcmp(argv[index], "--") != 0) { + throw std::runtime_error("containment helper target separator is missing"); + } + ++index; + if (result.protocol_fd < 0 || result.expected_parent < 0 || result.exec_path.empty() || + index >= argc) { + throw std::runtime_error("containment helper arguments are incomplete"); + } + for (; index < argc; ++index) { + result.target_argv.push_back(argv[index]); + } + result.target_argv.push_back(nullptr); + return result; +} + +#if defined(__linux__) + +constexpr uint32_t DIAGNOSTIC_MAGIC = 0x44535634U; +constexpr uint16_t DIAGNOSTIC_VERSION = 1; +constexpr size_t DIAGNOSTIC_STAGE_CAPACITY = 48; + +int query_supplementary_group_count(int * error_number) noexcept { + errno = 0; + const int count = getgroups(0, nullptr); + *error_number = count < 0 ? errno : 0; + return count; +} + +void require_zero_supplementary_groups(const char * context) { + int error_number = 0; + const int count = query_supplementary_group_count(&error_number); + if (count < 0) { + throw std::runtime_error( + std::string("cannot query ") + context + " supplementary groups: " + + std::strerror(error_number)); + } + if (count != 0) { + throw std::runtime_error( + std::string(context) + " requires zero supplementary groups; found " + + std::to_string(count)); + } +} + +struct failure_diagnostic { + uint32_t magic; + uint16_t version; + uint16_t stage_size; + int32_t error_number; + char stage[DIAGNOSTIC_STAGE_CAPACITY]; +}; + +static_assert(sizeof(failure_diagnostic) <= PIPE_BUF); + +void close_checked(int fd, const char * label); + +void report_failure(int fd, const char * stage, int error_number) noexcept { + const int saved_errno = errno; + failure_diagnostic record {}; + record.magic = DIAGNOSTIC_MAGIC; + record.version = DIAGNOSTIC_VERSION; + record.error_number = error_number; + while (record.stage_size < DIAGNOSTIC_STAGE_CAPACITY && + stage[record.stage_size] != '\0') { + record.stage[record.stage_size] = stage[record.stage_size]; + ++record.stage_size; + } + ssize_t written; + do { + written = write(fd, &record, sizeof(record)); + } while (written < 0 && errno == EINTR); + errno = saved_errno; +} + +[[noreturn]] void fail_stage( + int diagnostic_fd, + const char * stage, + int error_number, + int exit_code = 125) noexcept { + report_failure(diagnostic_fd, stage, error_number); + _exit(exit_code); +} + +std::string receive_failure_diagnostics(int fd) { + std::string result; + for (size_t count = 0; count < 8; ++count) { + failure_diagnostic record {}; + ssize_t size; + do { + size = read(fd, &record, sizeof(record)); + } while (size < 0 && errno == EINTR); + if (size < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + break; + } + if (size == 0) { + break; + } + if (size != static_cast(sizeof(record)) || + record.magic != DIAGNOSTIC_MAGIC || + record.version != DIAGNOSTIC_VERSION || + record.stage_size == 0 || + record.stage_size > DIAGNOSTIC_STAGE_CAPACITY) { + if (!result.empty()) { + result += "; "; + } + result += "invalid setup diagnostic"; + continue; + } + if (!result.empty()) { + result += "; "; + } + result += "stage="; + result.append(record.stage, record.stage_size); + result += " errno=" + std::to_string(record.error_number); + if (record.error_number != 0) { + result += " ("; + result += std::strerror(record.error_number); + result += ")"; + } + } + return result; +} + +[[noreturn]] void throw_setup_failure(int diagnostic_fd, const char * fallback) { + const std::string diagnostic = receive_failure_diagnostics(diagnostic_fd); + const int close_result = close(diagnostic_fd); + const int close_error = close_result == 0 ? 0 : errno; + std::string message = fallback; + if (!diagnostic.empty()) { + message += ": " + diagnostic; + } + if (close_error != 0) { + message += "; diagnostics-close errno=" + std::to_string(close_error); + message += " ("; + message += std::strerror(close_error); + message += ")"; + } + throw std::runtime_error(message); +} + +bool retained_fd( + int fd, + const std::vector & keep_fds, + int extra_fd, + int second_extra_fd = -1) { + if (fd >= 0 && fd <= STDERR_FILENO) { + return true; + } + if (fd == extra_fd || fd == second_extra_fd) { + return true; + } + for (int keep_fd : keep_fds) { + if (fd == keep_fd) { + return true; + } + } + return false; +} + +void close_checked(int fd, const char * label) { + if (close(fd) != 0) { + throw std::runtime_error(std::string("cannot close ") + label); + } +} + +void close_unneeded_fds( + const std::vector & keep_fds, + int extra_fd, + int second_extra_fd = -1) { + DIR * directory = opendir("/proc/self/fd"); + if (directory == nullptr) { + throw std::runtime_error("cannot open procfs descriptor directory"); + } + const int directory_fd = dirfd(directory); + while (dirent * entry = readdir(directory)) { + char * end = nullptr; + const long parsed = std::strtol(entry->d_name, &end, 10); + if (end == entry->d_name || *end != '\0' || parsed < 0 || + parsed > std::numeric_limits::max()) { + continue; + } + const int fd = static_cast(parsed); + if (fd != directory_fd && + !retained_fd(fd, keep_fds, extra_fd, second_extra_fd)) { + close_checked(fd, "unneeded descriptor"); + } + } + if (closedir(directory) != 0) { + throw std::runtime_error("cannot close procfs descriptor directory"); + } +} + +void verify_private_procfs() { + struct statfs filesystem {}; + if (statfs("/proc", &filesystem) != 0) { + throw std::runtime_error("cannot inspect private procfs"); + } + if (static_cast(filesystem.f_type) != + static_cast(PROC_SUPER_MAGIC)) { + throw std::runtime_error("private procfs has an unexpected filesystem type"); + } + char self_target[32] {}; + const ssize_t size = readlink("/proc/self", self_target, sizeof(self_target)); + if (size != 1 || self_target[0] != '1') { + throw std::runtime_error("private procfs is not bound to the target PID namespace"); + } +} + +void require_initial_signal_state() { + sigset_t mask; + if (sigprocmask(SIG_SETMASK, nullptr, &mask) != 0) { + throw std::runtime_error("cannot read containment helper signal mask"); + } + for (int signal_number = 1; signal_number < NSIG; ++signal_number) { + if (signal_number == SIGKILL || signal_number == SIGSTOP) { + continue; + } + struct sigaction action {}; + if (sigaction(signal_number, nullptr, &action) != 0) { + if (errno == EINVAL) { + continue; + } + throw std::runtime_error("cannot read containment helper signal disposition"); + } + if (sigismember(&mask, signal_number) != 1) { + throw std::runtime_error("containment helper inherited an unblocked signal"); + } + if (action.sa_handler != SIG_DFL) { + throw std::runtime_error("containment helper inherited a signal handler"); + } + } +} + +void unblock_default_signals() { + sigset_t empty; + sigemptyset(&empty); + if (sigprocmask(SIG_SETMASK, &empty, nullptr) != 0) { + throw std::runtime_error("cannot unblock containment helper signals"); + } +} + +void set_parent_death(pid_t expected_parent) { + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { + throw std::runtime_error("cannot set containment parent-death signal"); + } + if (getppid() != expected_parent) { + throw std::runtime_error("containment parent changed before lifecycle binding"); + } +} + +void require_parent_death(pid_t expected_parent) { + int parent_signal = 0; + if (prctl(PR_GET_PDEATHSIG, &parent_signal) != 0 || + parent_signal != SIGKILL || getppid() != expected_parent) { + throw std::runtime_error("containment parent-death binding changed"); + } +} + +void write_all(int fd, const void * data, size_t size) { + const char * cursor = static_cast(data); + while (size > 0) { + const ssize_t written = write(fd, cursor, size); + if (written < 0) { + if (errno == EINTR) { + continue; + } + throw std::runtime_error("containment protocol write failed"); + } + cursor += written; + size -= static_cast(written); + } +} + +void write_mapping_file(int process_directory, const char * name, const std::string & content) { + const int fd = openat(process_directory, name, O_WRONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) { + throw std::runtime_error(std::string("cannot open namespace ") + name); + } + write_all(fd, content.data(), content.size()); + close_checked(fd, "namespace mapping descriptor"); +} + +std::string receive_packet(int fd) { + char buffer[128]; + const ssize_t size = recv(fd, buffer, sizeof(buffer), 0); + if (size <= 0) { + throw std::runtime_error("containment protocol closed"); + } + return std::string(buffer, static_cast(size)); +} + +void send_packet(int fd, const char * packet) { + const size_t size = std::strlen(packet); + if (send(fd, packet, size, MSG_NOSIGNAL) != static_cast(size)) { + throw std::runtime_error("containment protocol send failed"); + } +} + +void send_descriptor(int fd, const char * packet, int descriptor) { + const size_t packet_size = std::strlen(packet); + iovec vector {const_cast(packet), packet_size}; + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int))] {}; + msghdr message {}; + message.msg_iov = &vector; + message.msg_iovlen = 1; + message.msg_control = control; + message.msg_controllen = sizeof(control); + cmsghdr * header = CMSG_FIRSTHDR(&message); + header->cmsg_level = SOL_SOCKET; + header->cmsg_type = SCM_RIGHTS; + header->cmsg_len = CMSG_LEN(sizeof(int)); + std::memcpy(CMSG_DATA(header), &descriptor, sizeof(descriptor)); + if (sendmsg(fd, &message, MSG_NOSIGNAL) != static_cast(packet_size)) { + throw std::runtime_error("cannot send containment descriptor"); + } +} + +int receive_descriptor(int fd, const char * expected_packet) { + char payload[32] {}; + iovec vector {payload, sizeof(payload)}; + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int))] {}; + msghdr message {}; + message.msg_iov = &vector; + message.msg_iovlen = 1; + message.msg_control = control; + message.msg_controllen = sizeof(control); + const ssize_t size = recvmsg(fd, &message, 0); + if (size != static_cast(std::strlen(expected_packet)) || + std::memcmp(payload, expected_packet, static_cast(size)) != 0 || + (message.msg_flags & (MSG_CTRUNC | MSG_TRUNC)) != 0) { + throw std::runtime_error("containment descriptor packet is invalid"); + } + cmsghdr * header = CMSG_FIRSTHDR(&message); + if (header == nullptr || + header->cmsg_level != SOL_SOCKET || + header->cmsg_type != SCM_RIGHTS || + header->cmsg_len != CMSG_LEN(sizeof(int)) || + CMSG_NXTHDR(&message, header) != nullptr) { + throw std::runtime_error("containment descriptor rights are invalid"); + } + int descriptor = -1; + std::memcpy(&descriptor, CMSG_DATA(header), sizeof(descriptor)); + if (descriptor < 0 || fcntl(descriptor, F_SETFD, FD_CLOEXEC) != 0) { + if (descriptor >= 0) { + close(descriptor); + } + throw std::runtime_error("cannot retain containment descriptor"); + } + return descriptor; +} + +bool pidfd_has_exited(int pidfd) { + pollfd descriptor {pidfd, POLLIN, 0}; + const int result = poll(&descriptor, 1, 0); + if (result < 0) { + throw std::runtime_error("cannot query containment pidfd"); + } + return result > 0 && (descriptor.revents & (POLLIN | POLLHUP | POLLERR)) != 0; +} + +int wait_status_exit_code(int status) { + if (WIFEXITED(status)) { + return WEXITSTATUS(status); + } + if (WIFSIGNALED(status)) { + return 128 + WTERMSIG(status); + } + return 125; +} + +void make_isolated_session() { + if (setsid() < 0 || getsid(0) != getpid() || getpgrp() != getpid()) { + throw std::runtime_error("cannot isolate containment session"); + } +} + +void protect_namespace_init() { + make_isolated_session(); + if (prctl(PR_SET_DUMPABLE, 0) != 0 || + prctl(PR_SET_PTRACER, 0) != 0 || + prctl(PR_GET_DUMPABLE) != 0) { + throw std::runtime_error("cannot protect target namespace init"); + } + require_parent_death(0); +} + +int read_cap_last_cap() { + const int fd = open("/proc/sys/kernel/cap_last_cap", O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) { + throw std::runtime_error("cannot open kernel capability limit"); + } + char buffer[32] {}; + const ssize_t size = read(fd, buffer, sizeof(buffer) - 1); + const int saved_errno = errno; + close_checked(fd, "kernel capability limit descriptor"); + if (size <= 0) { + errno = saved_errno; + throw std::runtime_error("cannot read kernel capability limit"); + } + char * end = nullptr; + errno = 0; + const long result = std::strtol(buffer, &end, 10); + if (errno != 0 || end == buffer || (*end != '\n' && *end != '\0') || + result < 0 || result > 63) { + throw std::runtime_error("kernel capability limit is invalid"); + } + return static_cast(result); +} + +uint32_t seccomp_audit_arch() { +#if defined(__x86_64__) + return AUDIT_ARCH_X86_64; +#elif defined(__aarch64__) + return AUDIT_ARCH_AARCH64; +#else +#error "Unsupported Linux architecture for DeepSeek V4.1 containment" +#endif +} + +void filter_statement(std::vector & filter, uint16_t code, uint32_t value) { + filter.push_back(sock_filter {code, 0, 0, value}); +} + +void filter_jump( + std::vector & filter, + uint16_t code, + uint32_t value, + uint8_t on_true, + uint8_t on_false) { + filter.push_back(sock_filter {code, on_true, on_false, value}); +} + +void filter_pid_argument( + std::vector & filter, + int syscall_number, + unsigned argument, + uint32_t denied_result) { + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, nr)); + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, static_cast(syscall_number), 0, 4); + filter_statement( + filter, + BPF_LD | BPF_W | BPF_ABS, + static_cast(offsetof(seccomp_data, args) + argument * sizeof(uint64_t))); + filter_jump(filter, BPF_JMP | BPF_JGE | BPF_K, 0x80000000U, 1, 0); + filter_jump(filter, BPF_JMP | BPF_JGT | BPF_K, 1, 1, 0); + filter_statement(filter, BPF_RET | BPF_K, denied_result); +} + +int install_target_seccomp_listener() { + std::vector denied_syscalls; +#define DSV41_DENY_SYSCALL(name) denied_syscalls.push_back(__NR_##name) +#if defined(__NR_ptrace) + DSV41_DENY_SYSCALL(ptrace); +#endif +#if defined(__NR_process_vm_readv) + DSV41_DENY_SYSCALL(process_vm_readv); +#endif +#if defined(__NR_process_vm_writev) + DSV41_DENY_SYSCALL(process_vm_writev); +#endif +#if defined(__NR_pidfd_send_signal) + DSV41_DENY_SYSCALL(pidfd_send_signal); +#endif +#if defined(__NR_setuid) + DSV41_DENY_SYSCALL(setuid); +#endif +#if defined(__NR_setgid) + DSV41_DENY_SYSCALL(setgid); +#endif +#if defined(__NR_setreuid) + DSV41_DENY_SYSCALL(setreuid); +#endif +#if defined(__NR_setregid) + DSV41_DENY_SYSCALL(setregid); +#endif +#if defined(__NR_setresuid) + DSV41_DENY_SYSCALL(setresuid); +#endif +#if defined(__NR_setresgid) + DSV41_DENY_SYSCALL(setresgid); +#endif +#if defined(__NR_setfsuid) + DSV41_DENY_SYSCALL(setfsuid); +#endif +#if defined(__NR_setfsgid) + DSV41_DENY_SYSCALL(setfsgid); +#endif +#if defined(__NR_setgroups) + DSV41_DENY_SYSCALL(setgroups); +#endif +#if defined(__NR_capset) + DSV41_DENY_SYSCALL(capset); +#endif +#if defined(__NR_setpgid) + DSV41_DENY_SYSCALL(setpgid); +#endif +#if defined(__NR_setsid) + DSV41_DENY_SYSCALL(setsid); +#endif +#if defined(__NR_setns) + DSV41_DENY_SYSCALL(setns); +#endif +#if defined(__NR_unshare) + DSV41_DENY_SYSCALL(unshare); +#endif +#undef DSV41_DENY_SYSCALL + + const int denied_prctl[] = { + PR_SET_PDEATHSIG, + PR_SET_DUMPABLE, + PR_SET_PTRACER, + PR_SET_SECUREBITS, + PR_SET_NO_NEW_PRIVS, + PR_CAPBSET_DROP, + PR_CAP_AMBIENT, + }; + constexpr uint32_t denied_result = SECCOMP_RET_USER_NOTIF; + std::vector filter; + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, arch)); + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, seccomp_audit_arch(), 1, 0); + filter_statement(filter, BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS); +#if defined(__NR_kill) + filter_pid_argument(filter, __NR_kill, 0, denied_result); +#endif +#if defined(__NR_tkill) + filter_pid_argument(filter, __NR_tkill, 0, denied_result); +#endif +#if defined(__NR_tgkill) + filter_pid_argument(filter, __NR_tgkill, 0, denied_result); + filter_pid_argument(filter, __NR_tgkill, 1, denied_result); +#endif +#if defined(__NR_rt_sigqueueinfo) + filter_pid_argument(filter, __NR_rt_sigqueueinfo, 0, denied_result); +#endif +#if defined(__NR_rt_tgsigqueueinfo) + filter_pid_argument(filter, __NR_rt_tgsigqueueinfo, 0, denied_result); + filter_pid_argument(filter, __NR_rt_tgsigqueueinfo, 1, denied_result); +#endif + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, nr)); + filter_jump( + filter, + BPF_JMP | BPF_JEQ | BPF_K, + __NR_prctl, + 0, + static_cast(2 + 2 * (sizeof(denied_prctl) / sizeof(denied_prctl[0])))); + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, args)); + for (int operation : denied_prctl) { + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, static_cast(operation), 0, 1); + filter_statement(filter, BPF_RET | BPF_K, denied_result); + } + filter_statement(filter, BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + filter_statement(filter, BPF_LD | BPF_W | BPF_ABS, offsetof(seccomp_data, nr)); + for (int syscall_number : denied_syscalls) { + filter_jump(filter, BPF_JMP | BPF_JEQ | BPF_K, static_cast(syscall_number), 0, 1); + filter_statement(filter, BPF_RET | BPF_K, denied_result); + } + filter_statement(filter, BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + if (filter.size() > std::numeric_limits::max()) { + throw std::runtime_error("target seccomp filter is too large"); + } + sock_fprog program { + static_cast(filter.size()), + filter.data(), + }; + const int listener = static_cast( + syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_NEW_LISTENER, &program)); + if (listener < 0 || fcntl(listener, F_SETFD, FD_CLOEXEC) != 0) { + if (listener >= 0) { + close(listener); + } + throw std::runtime_error("cannot install target seccomp filter"); + } + return listener; +} + +void require_eperm(long result, const char * label) { + if (result != -1 || errno != EPERM) { + throw std::runtime_error(std::string("target isolation did not deny ") + label); + } +} + +int drop_target_privileges() { + constexpr uid_t target_uid = 65534; + constexpr gid_t target_gid = 65534; + constexpr unsigned long securebits = + SECBIT_NOROOT | + SECBIT_NOROOT_LOCKED | + SECBIT_NO_SETUID_FIXUP | + SECBIT_NO_SETUID_FIXUP_LOCKED | + SECBIT_KEEP_CAPS_LOCKED | + SECBIT_NO_CAP_AMBIENT_RAISE | + SECBIT_NO_CAP_AMBIENT_RAISE_LOCKED; + const int cap_last_cap = read_cap_last_cap(); + if (prctl(PR_SET_SECUREBITS, securebits) != 0 || + setresgid(target_gid, target_gid, target_gid) != 0 || + setresuid(target_uid, target_uid, target_uid) != 0) { + throw std::runtime_error("cannot enter target non-root credentials"); + } + for (int capability = 0; capability <= cap_last_cap; ++capability) { + if (prctl(PR_CAPBSET_DROP, capability, 0, 0, 0) != 0) { + throw std::runtime_error("cannot drop target capability bounding set"); + } + } + if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0) != 0) { + throw std::runtime_error("cannot clear target ambient capabilities"); + } + __user_cap_header_struct header {}; + header.version = _LINUX_CAPABILITY_VERSION_3; + __user_cap_data_struct data[2] {}; + if (syscall(SYS_capset, &header, data) != 0 || + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + throw std::runtime_error("cannot clear target capabilities"); + } + __user_cap_data_struct verified[2] {}; + if (syscall(SYS_capget, &header, verified) != 0) { + throw std::runtime_error("cannot verify target capabilities"); + } + for (const __user_cap_data_struct & entry : verified) { + if (entry.effective != 0 || entry.permitted != 0 || entry.inheritable != 0) { + throw std::runtime_error("target capability sets are not empty"); + } + } + for (int capability = 0; capability <= cap_last_cap; ++capability) { + if (prctl(PR_CAPBSET_READ, capability, 0, 0, 0) != 0 || + prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_IS_SET, capability, 0, 0) != 0) { + throw std::runtime_error("target retained a bounded or ambient capability"); + } + } + if (getuid() != target_uid || geteuid() != target_uid || + getgid() != target_gid || getegid() != target_gid || + getgroups(0, nullptr) != 0 || + prctl(PR_GET_SECUREBITS) != static_cast(securebits) || + prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1 || + getsid(0) != getpid() || getpgrp() != getpid()) { + throw std::runtime_error("target privilege isolation verification failed"); + } + const int listener = install_target_seccomp_listener(); + if (prctl(PR_GET_SECCOMP, 0, 0, 0, 0) != SECCOMP_MODE_FILTER) { + close(listener); + throw std::runtime_error("target seccomp verification failed"); + } + return listener; +} + +void verify_target_attack_denials() { + errno = 0; + require_eperm(syscall(SYS_ptrace, PTRACE_ATTACH, 1, nullptr, nullptr), "ptrace"); +#if defined(SYS_process_vm_readv) + errno = 0; + require_eperm(syscall(SYS_process_vm_readv, 1, nullptr, 0, nullptr, 0, 0), "process_vm_readv"); +#endif +#if defined(SYS_process_vm_writev) + errno = 0; + require_eperm(syscall(SYS_process_vm_writev, 1, nullptr, 0, nullptr, 0, 0), "process_vm_writev"); +#endif + errno = 0; + require_eperm(kill(1, SIGSTOP), "namespace init signaling"); + errno = 0; + require_eperm(kill(-getpgrp(), 0), "process-group signaling"); + if (kill(getpid(), 0) != 0) { + throw std::runtime_error("target isolation blocked local signaling"); + } + errno = 0; + require_eperm(prctl(PR_SET_PDEATHSIG, 0), "parent-death mutation"); + errno = 0; + require_eperm(setresuid(0, 0, 0), "UID regain"); + errno = 0; + require_eperm(setpgid(0, 0), "process-group mutation"); + errno = 0; + require_eperm(setsid(), "session mutation"); + __user_cap_header_struct header {}; + header.version = _LINUX_CAPABILITY_VERSION_3; + __user_cap_data_struct data[2] {}; + errno = 0; + require_eperm(syscall(SYS_capset, &header, data), "capability regain"); +} + +struct seccomp_notification { + bool present; + uint64_t id; + seccomp_data data; +}; + +seccomp_notification receive_seccomp_notification(int listener, bool nonblocking) { + seccomp_notif_sizes sizes {}; + if (syscall(SYS_seccomp, SECCOMP_GET_NOTIF_SIZES, 0, &sizes) != 0 || + sizes.seccomp_notif < sizeof(seccomp_notif) || + sizes.seccomp_notif_resp < sizeof(seccomp_notif_resp)) { + throw std::runtime_error("cannot query target seccomp notification sizes"); + } + std::vector request_buffer( + (sizes.seccomp_notif + sizeof(uint64_t) - 1) / sizeof(uint64_t)); + seccomp_notif * request = reinterpret_cast(request_buffer.data()); + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, request) != 0) { + if (nonblocking && (errno == EAGAIN || errno == ENOENT)) { + return seccomp_notification {false, 0, {}}; + } + throw std::runtime_error("cannot receive target seccomp notification"); + } + if (request->flags != 0 || + ioctl(listener, SECCOMP_IOCTL_NOTIF_ID_VALID, &request->id) != 0) { + throw std::runtime_error("target seccomp notification identity is invalid"); + } + return seccomp_notification {true, request->id, request->data}; +} + +void deny_seccomp_notification(int listener, const seccomp_notification & notification) { + seccomp_notif_resp response {}; + response.id = notification.id; + response.error = -EPERM; + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &response) != 0) { + throw std::runtime_error("cannot deny target seccomp notification"); + } +} + +seccomp_notification expect_seccomp_notification( + int listener, + int syscall_number, + const char * label) { + const seccomp_notification notification = receive_seccomp_notification(listener, false); + if (!notification.present || notification.data.nr != syscall_number) { + throw std::runtime_error(std::string("unexpected target isolation probe: ") + label); + } + deny_seccomp_notification(listener, notification); + return notification; +} + +void verify_target_isolation_probes(int listener) { + expect_seccomp_notification(listener, SYS_ptrace, "ptrace"); +#if defined(SYS_process_vm_readv) + expect_seccomp_notification(listener, SYS_process_vm_readv, "process_vm_readv"); +#endif +#if defined(SYS_process_vm_writev) + expect_seccomp_notification(listener, SYS_process_vm_writev, "process_vm_writev"); +#endif + seccomp_notification notification = expect_seccomp_notification(listener, SYS_kill, "PID 1 signaling"); + if (notification.data.args[0] != 1 || notification.data.args[1] != SIGSTOP) { + throw std::runtime_error("target PID 1 signal probe is invalid"); + } + notification = expect_seccomp_notification(listener, SYS_kill, "process-group signaling"); + if (static_cast(notification.data.args[0]) >= 0) { + throw std::runtime_error("target process-group signal probe is invalid"); + } + notification = expect_seccomp_notification(listener, SYS_prctl, "parent-death mutation"); + if (notification.data.args[0] != PR_SET_PDEATHSIG) { + throw std::runtime_error("target parent-death probe is invalid"); + } + expect_seccomp_notification(listener, SYS_setresuid, "UID regain"); + expect_seccomp_notification(listener, SYS_setpgid, "process-group mutation"); + expect_seccomp_notification(listener, SYS_setsid, "session mutation"); + expect_seccomp_notification(listener, SYS_capset, "capability regain"); +} + +class namespace_owner { +public: + namespace_owner(pid_t pid, int pidfd) : pid_(pid), pidfd_(pidfd) { + } + + ~namespace_owner() { + if (reaped_) { + return; + } + if (pidfd_ >= 0) { + syscall(SYS_pidfd_send_signal, pidfd_, SIGKILL, nullptr, 0); + } + while (waitpid(pid_, nullptr, 0) < 0 && errno == EINTR) { + } + if (pidfd_ >= 0) { + close(pidfd_); + } + } + + int pidfd() const { + return pidfd_; + } + + int wait() { + int status = 0; + while (waitpid(pid_, &status, 0) < 0) { + if (errno != EINTR) { + throw std::runtime_error("cannot reap target PID namespace"); + } + } + reaped_ = true; + return status; + } + + void close_pidfd() { + const int descriptor = pidfd_; + pidfd_ = -1; + if (close(descriptor) != 0) { + throw std::runtime_error("cannot close namespace pidfd"); + } + } + +private: + pid_t pid_; + int pidfd_; + bool reaped_ = false; +}; + +int wait_for_isolated_target(namespace_owner & target, int listener) { + const int flags = fcntl(listener, F_GETFL); + if (flags < 0 || fcntl(listener, F_SETFL, flags | O_NONBLOCK) != 0) { + throw std::runtime_error("cannot configure target seccomp listener"); + } + pollfd descriptors[2] { + {listener, POLLIN, 0}, + {target.pidfd(), POLLIN, 0}, + }; + while (true) { + const int result = poll(descriptors, 2, -1); + if (result < 0) { + if (errno == EINTR) { + continue; + } + throw std::runtime_error("cannot monitor isolated target"); + } + if ((descriptors[0].revents & (POLLIN | POLLHUP | POLLERR)) != 0) { + const seccomp_notification notification = receive_seccomp_notification(listener, true); + if (notification.present) { + throw std::runtime_error("target attempted a forbidden lifecycle operation"); + } + } + if ((descriptors[1].revents & (POLLIN | POLLHUP | POLLERR)) != 0) { + const seccomp_notification notification = receive_seccomp_notification(listener, true); + if (notification.present) { + throw std::runtime_error("target attempted a forbidden lifecycle operation"); + } + return target.wait(); + } + } +} + +[[noreturn]] void run_target_bootstrap( + int mapping_fd, + int ready_fd, + int security_fd, + int namespace_ready_fd, + int diagnostic_fd, + const options & config) { + const char * stage = "target-parent-death"; + try { + errno = 0; + set_parent_death(1); + stage = "target-session"; + errno = 0; + make_isolated_session(); + stage = "target-namespace-ready-close"; + errno = 0; + close_checked(namespace_ready_fd, "target namespace readiness descriptor"); + stage = "target-bound"; + errno = 0; + write_all(ready_fd, "B", 1); + stage = "target-mapping-read"; + char mapped = 0; + ssize_t mapped_size; + do { + mapped_size = read(mapping_fd, &mapped, 1); + } while (mapped_size < 0 && errno == EINTR); + if (mapped_size != 1 || mapped != 'M') { + fail_stage(diagnostic_fd, stage, mapped_size < 0 ? errno : 0); + } + stage = "target-mapping-close"; + errno = 0; + close_checked(mapping_fd, "target mapping descriptor"); + stage = "target-groups-verify"; + int groups_error = 0; + if (query_supplementary_group_count(&groups_error) != 0) { + fail_stage(diagnostic_fd, stage, groups_error); + } + stage = "target-privilege-drop"; + errno = 0; + const int listener = drop_target_privileges(); + stage = "target-filter-send"; + errno = 0; + send_descriptor(security_fd, "FILTER", listener); + stage = "target-listener-close"; + errno = 0; + close_checked(listener, "target seccomp listener"); + stage = "target-isolation-probes"; + errno = 0; + verify_target_attack_denials(); + stage = "target-verified-send"; + errno = 0; + send_packet(security_fd, "VERIFIED"); + stage = "target-go-read"; + errno = 0; + if (receive_packet(security_fd) != "GO") { + fail_stage(diagnostic_fd, stage, 0); + } + stage = "target-security-close"; + errno = 0; + close_checked(security_fd, "target security descriptor"); + stage = "target-parent-death-verify"; + errno = 0; + require_parent_death(1); + stage = "target-fd-close"; + errno = 0; + close_unneeded_fds(config.keep_fds, ready_fd, diagnostic_fd); + stage = "target-isolation-ready"; + errno = 0; + write_all(ready_fd, "I", 1); + stage = "target-ready-close"; + errno = 0; + close_checked(ready_fd, "target readiness descriptor"); + stage = "target-exec"; + errno = 0; + execve(config.exec_path.c_str(), config.target_argv.data(), environ); + fail_stage(diagnostic_fd, stage, errno, 127); + } catch (...) { + fail_stage(diagnostic_fd, stage, errno); + } +} + +[[noreturn]] void run_namespace_init( + int release_fd, + int ready_fd, + int mapping_fd, + int diagnostic_fd, + int protocol_fd, + int helper_pidfd, + const options & config) { + const char * stage = "namespace-parent-death"; + try { + errno = 0; + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-parent-identity"; + errno = 0; + if (getppid() != 0 || pidfd_has_exited(helper_pidfd)) { + fail_stage(diagnostic_fd, stage, 0); + } + stage = "namespace-helper-pidfd-close"; + errno = 0; + close_checked(helper_pidfd, "namespace helper pidfd"); + stage = "namespace-protocol-close"; + errno = 0; + close_checked(protocol_fd, "namespace protocol descriptor"); + stage = "namespace-bound"; + errno = 0; + write_all(ready_fd, "B", 1); + stage = "namespace-mapping-read"; + char mapped = 0; + ssize_t mapped_size; + do { + mapped_size = read(mapping_fd, &mapped, 1); + } while (mapped_size < 0 && errno == EINTR); + if (mapped_size != 1 || mapped != 'M') { + fail_stage(diagnostic_fd, stage, mapped_size < 0 ? errno : 0); + } + stage = "namespace-mapping-close"; + errno = 0; + close_checked(mapping_fd, "namespace mapping descriptor"); + stage = "namespace-groups-verify"; + int groups_error = 0; + if (query_supplementary_group_count(&groups_error) != 0) { + fail_stage(diagnostic_fd, stage, groups_error); + } + stage = "namespace-setresgid"; + errno = 0; + if (setresgid(0, 0, 0) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-setresuid"; + errno = 0; + if (setresuid(0, 0, 0) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-mount-private"; + errno = 0; + if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-unmount-proc"; + errno = 0; + if (umount2("/proc", MNT_DETACH) != 0 && errno != EINVAL) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-mount-proc"; + errno = 0; + if (mount("proc", "/proc", "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC, nullptr) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-verify-proc"; + errno = 0; + verify_private_procfs(); + stage = "namespace-protect-init"; + errno = 0; + protect_namespace_init(); + stage = "namespace-ready"; + errno = 0; + write_all(ready_fd, "R", 1); + stage = "namespace-release-read"; + char release = 0; + ssize_t received; + do { + received = read(release_fd, &release, 1); + } while (received < 0 && errno == EINTR); + if (received != 1 || release != 'X') { + fail_stage(diagnostic_fd, stage, received < 0 ? errno : 0); + } + stage = "namespace-release-close"; + errno = 0; + close_checked(release_fd, "namespace release descriptor"); + int target_mapping_pipe[2] {-1, -1}; + int target_ready_pipe[2] {-1, -1}; + int target_security[2] {-1, -1}; + stage = "namespace-target-mapping-pipe"; + errno = 0; + if (pipe2(target_mapping_pipe, O_CLOEXEC) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-target-ready-pipe"; + errno = 0; + if (pipe2(target_ready_pipe, O_CLOEXEC) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-target-security-socket"; + errno = 0; + if (socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, target_security) != 0) { + fail_stage(diagnostic_fd, stage, errno); + } + int target_pidfd = -1; + clone_args target_arguments {}; + target_arguments.flags = CLONE_NEWUSER | CLONE_PIDFD; + target_arguments.pidfd = reinterpret_cast(&target_pidfd); + target_arguments.exit_signal = SIGCHLD; + stage = "namespace-target-clone"; + errno = 0; + const pid_t target_pid = static_cast( + syscall(SYS_clone3, &target_arguments, sizeof(target_arguments))); + if (target_pid < 0) { + fail_stage(diagnostic_fd, stage, errno); + } + if (target_pid == 0) { + stage = "target-handoff-close"; + errno = 0; + close_checked(target_mapping_pipe[1], "target mapping writer"); + close_checked(target_ready_pipe[0], "target readiness reader"); + close_checked(target_security[0], "target security supervisor descriptor"); + run_target_bootstrap( + target_mapping_pipe[0], target_ready_pipe[1], + target_security[1], ready_fd, diagnostic_fd, config); + } + namespace_owner owned_target(target_pid, target_pidfd); + stage = "namespace-target-parent-close"; + errno = 0; + close_checked(target_mapping_pipe[0], "namespace target mapping reader"); + close_checked(target_ready_pipe[1], "namespace target readiness writer"); + close_checked(target_security[1], "namespace target security descriptor"); + stage = "namespace-target-bound-read"; + char target_ready = 0; + ssize_t target_ready_size; + do { + target_ready_size = read(target_ready_pipe[0], &target_ready, 1); + } while (target_ready_size < 0 && errno == EINTR); + if (target_ready_size != 1 || target_ready != 'B') { + fail_stage(diagnostic_fd, stage, target_ready_size < 0 ? errno : 0); + } + stage = "namespace-target-process-open"; + errno = 0; + const std::string target_process_path = "/proc/" + std::to_string(target_pid); + const int target_process_directory = open( + target_process_path.c_str(), O_PATH | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); + if (target_process_directory < 0 || pidfd_has_exited(target_pidfd)) { + fail_stage( + diagnostic_fd, stage, + target_process_directory < 0 ? errno : 0); + } + stage = "namespace-target-map-setgroups"; + errno = 0; + write_mapping_file(target_process_directory, "setgroups", "deny\n"); + stage = "namespace-target-map-uid"; + errno = 0; + write_mapping_file(target_process_directory, "uid_map", "65534 0 1\n"); + stage = "namespace-target-map-gid"; + errno = 0; + write_mapping_file(target_process_directory, "gid_map", "65534 0 1\n"); + stage = "namespace-target-process-close"; + errno = 0; + close_checked(target_process_directory, "target process directory"); + stage = "namespace-target-mapping-release"; + errno = 0; + write_all(target_mapping_pipe[1], "M", 1); + stage = "namespace-target-mapping-close"; + errno = 0; + close_checked(target_mapping_pipe[1], "namespace target mapping writer"); + stage = "namespace-target-filter-receive"; + errno = 0; + const int target_listener = receive_descriptor(target_security[0], "FILTER"); + stage = "namespace-target-probe-verify"; + errno = 0; + verify_target_isolation_probes(target_listener); + stage = "namespace-target-verified-read"; + errno = 0; + if (receive_packet(target_security[0]) != "VERIFIED") { + fail_stage(diagnostic_fd, stage, 0); + } + stage = "namespace-parent-death-verify"; + errno = 0; + require_parent_death(0); + stage = "namespace-identity-verify"; + errno = 0; + if (prctl(PR_GET_DUMPABLE) != 0 || + getsid(0) != getpid() || getpgrp() != getpid()) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-target-go"; + errno = 0; + send_packet(target_security[0], "GO"); + stage = "namespace-target-security-close"; + errno = 0; + close_checked(target_security[0], "namespace target security supervisor descriptor"); + stage = "namespace-target-isolation-read"; + do { + target_ready_size = read(target_ready_pipe[0], &target_ready, 1); + } while (target_ready_size < 0 && errno == EINTR); + stage = "namespace-target-ready-close"; + errno = 0; + close_checked(target_ready_pipe[0], "namespace target readiness reader"); + stage = "namespace-parent-death-reverify"; + errno = 0; + require_parent_death(0); + stage = "namespace-target-isolation-verify"; + errno = 0; + if (prctl(PR_GET_DUMPABLE) != 0 || + getsid(0) != getpid() || getpgrp() != getpid() || + target_ready_size != 1 || target_ready != 'I') { + fail_stage(diagnostic_fd, stage, 0); + } + stage = "namespace-isolation-ready"; + errno = 0; + write_all(ready_fd, "I", 1); + stage = "namespace-target-wait"; + errno = 0; + const int target_status = wait_for_isolated_target(owned_target, target_listener); + stage = "namespace-listener-close"; + errno = 0; + close_checked(target_listener, "namespace target seccomp listener"); + stage = "namespace-target-pidfd-close"; + errno = 0; + owned_target.close_pidfd(); + stage = "namespace-descendant-kill"; + errno = 0; + if (kill(-1, SIGKILL) != 0 && errno != ESRCH) { + fail_stage(diagnostic_fd, stage, errno); + } + stage = "namespace-descendant-reap"; + errno = 0; + while (true) { + const pid_t reaped = waitpid(-1, nullptr, 0); + if (reaped > 0 || (reaped < 0 && errno == EINTR)) { + continue; + } + if (reaped < 0 && errno == ECHILD) { + break; + } + fail_stage(diagnostic_fd, stage, reaped < 0 ? errno : 0); + } + stage = "namespace-complete"; + errno = 0; + write_all(ready_fd, "C", 1); + stage = "namespace-ready-close"; + errno = 0; + close_checked(ready_fd, "namespace readiness descriptor"); + _exit(wait_status_exit_code(target_status)); + } catch (...) { + fail_stage(diagnostic_fd, stage, errno); + } +} + +int run_linux_helper(options config) { + require_zero_supplementary_groups("containment launcher"); + require_initial_signal_state(); + set_parent_death(config.expected_parent); + close_unneeded_fds(config.keep_fds, config.protocol_fd); + unblock_default_signals(); + send_packet(config.protocol_fd, "READY"); + if (receive_packet(config.protocol_fd) != "PREPARE") { + throw std::runtime_error("containment PREPARE packet is invalid"); + } + + int release_pipe[2] {-1, -1}; + if (pipe2(release_pipe, O_CLOEXEC) != 0) { + throw std::runtime_error("cannot create namespace release pipe"); + } + int ready_pipe[2] {-1, -1}; + if (pipe2(ready_pipe, O_CLOEXEC) != 0) { + close(release_pipe[0]); + close(release_pipe[1]); + throw std::runtime_error("cannot create namespace readiness pipe"); + } + int mapping_pipe[2] {-1, -1}; + if (pipe2(mapping_pipe, O_CLOEXEC) != 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + throw std::runtime_error("cannot create namespace mapping pipe"); + } + int diagnostic_pipe[2] {-1, -1}; + if (pipe2(diagnostic_pipe, O_CLOEXEC | O_NONBLOCK) != 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + close(mapping_pipe[0]); + close(mapping_pipe[1]); + throw std::runtime_error("cannot create namespace diagnostic pipe"); + } + const int helper_pidfd = static_cast(syscall(SYS_pidfd_open, getpid(), 0)); + if (helper_pidfd < 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + close(mapping_pipe[0]); + close(mapping_pipe[1]); + close(diagnostic_pipe[0]); + close(diagnostic_pipe[1]); + throw std::runtime_error("cannot open stable helper identity"); + } + int namespace_pidfd = -1; + clone_args arguments {}; + arguments.flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_PIDFD; + arguments.pidfd = reinterpret_cast(&namespace_pidfd); + arguments.exit_signal = SIGCHLD; + const pid_t namespace_init = static_cast( + syscall(SYS_clone3, &arguments, sizeof(arguments))); + if (namespace_init < 0) { + close(release_pipe[0]); + close(release_pipe[1]); + close(ready_pipe[0]); + close(ready_pipe[1]); + close(mapping_pipe[0]); + close(mapping_pipe[1]); + close(diagnostic_pipe[0]); + close(diagnostic_pipe[1]); + close(helper_pidfd); + throw std::runtime_error(std::string("cannot create target PID namespace: ") + std::strerror(errno)); + } + if (namespace_init == 0) { + if (close(release_pipe[1]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-release-writer-close", + errno); + } + if (close(ready_pipe[0]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-ready-reader-close", + errno); + } + if (close(mapping_pipe[1]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-mapping-writer-close", + errno); + } + if (close(diagnostic_pipe[0]) != 0) { + fail_stage( + diagnostic_pipe[1], + "namespace-diagnostic-reader-close", + errno); + } + run_namespace_init( + release_pipe[0], ready_pipe[1], mapping_pipe[0], diagnostic_pipe[1], + config.protocol_fd, helper_pidfd, config); + } + namespace_owner owned_namespace(namespace_init, namespace_pidfd); + close_checked(release_pipe[0], "helper release reader"); + close_checked(ready_pipe[1], "helper readiness writer"); + close_checked(mapping_pipe[0], "helper mapping reader"); + close_checked(diagnostic_pipe[1], "helper diagnostic writer"); + char ready = 0; + ssize_t ready_size; + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + if (ready_size != 1 || ready != 'B') { + throw_setup_failure( + diagnostic_pipe[0], + "target PID namespace did not bind helper lifetime"); + } + const std::string process_path = "/proc/" + std::to_string(namespace_init); + const int process_directory = open( + process_path.c_str(), O_PATH | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); + if (process_directory < 0 || pidfd_has_exited(namespace_pidfd)) { + if (process_directory >= 0) { + close_checked(process_directory, "namespace process directory"); + } + throw std::runtime_error("cannot retain target namespace mapping identity"); + } + write_mapping_file(process_directory, "setgroups", "deny\n"); + write_mapping_file( + process_directory, "uid_map", + "0 " + std::to_string(getuid()) + " 1\n"); + write_mapping_file( + process_directory, "gid_map", + "0 " + std::to_string(getgid()) + " 1\n"); + close_checked(process_directory, "namespace process directory"); + write_all(mapping_pipe[1], "M", 1); + close_checked(mapping_pipe[1], "helper mapping writer"); + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + close_checked(helper_pidfd, "helper self pidfd"); + if (ready_size != 1 || ready != 'R') { + throw_setup_failure( + diagnostic_pipe[0], + "target PID namespace setup did not complete"); + } + send_descriptor(config.protocol_fd, "PREPARED", owned_namespace.pidfd()); + if (receive_packet(config.protocol_fd) != "EXEC") { + throw std::runtime_error("containment EXEC packet is invalid"); + } + write_all(release_pipe[1], "X", 1); + close_checked(release_pipe[1], "helper release writer"); + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + if (ready_size != 1 || ready != 'I') { + throw_setup_failure( + diagnostic_pipe[0], + "target privilege isolation did not complete"); + } + send_packet(config.protocol_fd, "RELEASED"); + const int status = owned_namespace.wait(); + do { + ready_size = read(ready_pipe[0], &ready, 1); + } while (ready_size < 0 && errno == EINTR); + close_checked(ready_pipe[0], "helper readiness reader"); + if (ready_size != 1 || ready != 'C') { + throw_setup_failure( + diagnostic_pipe[0], + "target namespace teardown did not complete"); + } + close_checked(diagnostic_pipe[0], "helper diagnostics reader"); + owned_namespace.close_pidfd(); + send_packet(config.protocol_fd, "COMPLETE"); + return wait_status_exit_code(status); +} + +#endif + +} + +int main(int argc, char ** argv) { + try { + if (argc == 2 && std::strcmp(argv[1], "--version") == 0) { + std::cout << "deepseek-v41-containment-helper " << DSV41_BUILD_REVISION << '\n'; + return 0; + } +#if defined(__linux__) + if (argc == 2 && std::strcmp(argv[1], "--check-launcher-groups") == 0) { + require_zero_supplementary_groups("containment launcher"); + std::cout << "supplementary-groups=0\n"; + return 0; + } + return run_linux_helper(parse_options(argc, argv)); +#else + (void) argc; + (void) argv; + std::cerr << "Linux PID namespace containment is unavailable on this platform\n"; + return 125; +#endif + } catch (const std::exception & error) { + std::cerr << error.what() << '\n'; + return 125; + } +} diff --git a/tools/deepseek-v41-trace/llama-trace.cpp b/tools/deepseek-v41-trace/llama-trace.cpp new file mode 100644 index 000000000000..3a25fc2c5753 --- /dev/null +++ b/tools/deepseek-v41-trace/llama-trace.cpp @@ -0,0 +1,1866 @@ +#include "arg.h" +#include "build-info.h" +#include "common.h" +#include "ggml-backend.h" +#include "ggml.h" +extern "C" { +#include "hash/sha256/sha256.h" +} +#include "llama.h" +#include "llama-ext.h" +#include "host-attestation.h" +#include "trace-components.h" +#include "dsv41-runtime-receipt.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#include +#else +#include +#if defined(__APPLE__) +#include +#endif +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#endif +#endif + +#if !defined(_WIN32) +#include +#endif + +namespace fs = std::filesystem; +using json = nlohmann::json; + +static constexpr int TRACE_VERSION = 2; +static constexpr const char * BUILD_REVISION = DSV41_BUILD_REVISION; +#if defined(__linux__) +static constexpr const char * WATCHDOG_SCRIPT_SHA256 = + "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f"; +#endif + +static std::string sha256_hex(const unsigned char digest[SHA256_DIGEST_SIZE]) { + std::ostringstream stream; + stream << std::hex << std::setfill('0'); + for (size_t i = 0; i < SHA256_DIGEST_SIZE; ++i) { + stream << std::setw(2) << static_cast(digest[i]); + } + return stream.str(); +} + +static std::string sha256_data(const void * data, size_t size) { + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_hash(digest, static_cast(data), size); + return sha256_hex(digest); +} + +static std::string sha256_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open for SHA-256: " + path.string()); + } + + sha256_t state; + sha256_init(&state); + std::vector buffer(1024 * 1024); + while (input) { + input.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + const std::streamsize count = input.gcount(); + if (count > 0) { + sha256_update(&state, buffer.data(), static_cast(count)); + } + } + if (!input.eof()) { + throw std::runtime_error("failed while hashing: " + path.string()); + } + + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_final(&state, digest); + return sha256_hex(digest); +} + +static std::vector read_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input) { + throw std::runtime_error("cannot open: " + path.string()); + } + const std::streamsize size = input.tellg(); + if (size < 0) { + throw std::runtime_error("cannot determine file size: " + path.string()); + } + std::vector result(static_cast(size)); + input.seekg(0); + if (size != 0 && !input.read(reinterpret_cast(result.data()), size)) { + throw std::runtime_error("cannot read: " + path.string()); + } + return result; +} + +static fs::path canonical_path(const fs::path & path, const char * label) { + try { + return fs::canonical(path); + } catch (const fs::filesystem_error & error) { + throw std::runtime_error(std::string("cannot resolve ") + label + ": " + error.what()); + } +} + +static fs::path current_executable_path() { +#if defined(_WIN32) + std::vector buffer(32768); + const DWORD size = GetModuleFileNameW(nullptr, buffer.data(), static_cast(buffer.size())); + if (size == 0 || size >= buffer.size()) { + throw std::runtime_error("cannot query current executable path"); + } + return canonical_path(fs::path(std::wstring(buffer.data(), size)), "current executable"); +#elif defined(__APPLE__) + uint32_t size = 0; + if (_NSGetExecutablePath(nullptr, &size) != -1 || size == 0) { + throw std::runtime_error("cannot query current executable path size"); + } + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) { + throw std::runtime_error("cannot query current executable path"); + } + return canonical_path(buffer.data(), "current executable"); +#elif defined(__linux__) + return canonical_path("/proc/self/exe", "current executable"); +#else +#error unsupported platform +#endif +} + +static fs::path module_path(const void * address) { +#if defined(_WIN32) + HMODULE module = nullptr; + if (!GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(address), + &module)) { + throw std::runtime_error("cannot identify loaded runtime module"); + } + std::vector buffer(32768); + const DWORD size = GetModuleFileNameW(module, buffer.data(), static_cast(buffer.size())); + if (size == 0 || size >= buffer.size()) { + throw std::runtime_error("cannot query loaded runtime module path"); + } + return canonical_path(fs::path(std::wstring(buffer.data(), size)), "loaded runtime module"); +#else + Dl_info info = {}; + if (dladdr(address, &info) == 0 || info.dli_fname == nullptr || info.dli_fname[0] == '\0') { + throw std::runtime_error("cannot identify loaded runtime module"); + } + return canonical_path(info.dli_fname, "loaded runtime module"); +#endif +} + +template +static const void * function_address(T function) { + return reinterpret_cast(reinterpret_cast(function)); +} + +static bool is_project_runtime_library(const fs::path & path) { + std::string name = path.filename().string(); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { + return static_cast(std::tolower(value)); + }); +#if defined(_WIN32) + return name.rfind("llama", 0) == 0 || name.rfind("ggml", 0) == 0 || + name.rfind("libllama", 0) == 0 || name.rfind("libggml", 0) == 0; +#else + return name.rfind("libllama", 0) == 0 || name.rfind("libggml", 0) == 0 || + name.rfind("ggml", 0) == 0; +#endif +} + +static fs::path loaded_image_path(const fs::path & path) { + if (fs::exists(path)) { + return canonical_path(path, "loaded runtime module"); + } + if (!path.is_absolute()) { + throw std::runtime_error("loaded runtime module path is not absolute: " + path.string()); + } + return path.lexically_normal(); +} + +static bool is_trusted_system_runtime_path(const fs::path & path) { +#if defined(_WIN32) + std::vector buffer(32768); + const UINT size = GetSystemDirectoryW(buffer.data(), static_cast(buffer.size())); + if (size == 0 || size >= buffer.size()) { + throw std::runtime_error("cannot query the Windows system runtime directory"); + } + return path.parent_path() == canonical_path( + fs::path(std::wstring(buffer.data(), size)), "Windows system runtime directory"); +#elif defined(__APPLE__) + return path.string().rfind("/usr/lib/", 0) == 0 || + path.string().rfind("/System/Library/", 0) == 0; +#elif defined(__linux__) + const std::string value = path.string(); + std::error_code error; + const fs::path rocm_root = fs::canonical("/opt/rocm", error); + if (!error) { + const std::string prefix = rocm_root.string() + "/"; + if (value.rfind(prefix, 0) == 0) { + for (fs::path current = path; current != rocm_root.parent_path(); current = current.parent_path()) { + struct stat status = {}; + if (::stat(current.c_str(), &status) != 0 || status.st_uid != 0 || + (status.st_mode & (S_IWGRP | S_IWOTH)) != 0) { + return false; + } + if (current == rocm_root) { + return true; + } + } + } + } + return value.rfind("/lib/", 0) == 0 || + value.rfind("/lib64/", 0) == 0 || + value.rfind("/usr/lib/", 0) == 0 || + value.rfind("/usr/lib64/", 0) == 0; +#else + (void) path; + return false; +#endif +} + +static fs::path runtime_library_directory(const fs::path & executable); + +#if defined(__APPLE__) +static std::array protected_interval_images = {}; +static std::atomic protected_interval_image_count = 0; +static std::atomic protected_interval_active = false; + +static void record_loaded_image(const mach_header * header, intptr_t) { + if (!protected_interval_active.load(std::memory_order_relaxed)) { + return; + } + const size_t index = protected_interval_image_count.fetch_add(1, std::memory_order_relaxed); + if (index < protected_interval_images.size()) { + protected_interval_images[index] = header; + } +} + +static void begin_loader_monitor() { + static const bool registered = []() { + _dyld_register_func_for_add_image(record_loaded_image); + return true; + }(); + (void) registered; + protected_interval_image_count.store(0, std::memory_order_relaxed); + protected_interval_active.store(true, std::memory_order_release); +} + +static void end_loader_monitor(const fs::path & executable) { + protected_interval_active.store(false, std::memory_order_release); + const size_t count = protected_interval_image_count.load(std::memory_order_relaxed); + if (count > protected_interval_images.size()) { + throw std::runtime_error("too many loader image additions during protected trace generation"); + } + const fs::path library_directory = + canonical_path(runtime_library_directory(executable), "runtime library directory"); + for (size_t index = 0; index < count; ++index) { + Dl_info info = {}; + if (dladdr(protected_interval_images[index], &info) == 0 || + info.dli_fname == nullptr || info.dli_fname[0] == '\0') { + throw std::runtime_error("cannot identify loader image added during protected trace generation"); + } + const fs::path image = loaded_image_path(info.dli_fname); + if (image.parent_path() == executable.parent_path() || + image.parent_path() == library_directory || + is_project_runtime_library(image) || + !is_trusted_system_runtime_path(image)) { + throw std::runtime_error( + "runtime module was added during protected trace generation: " + image.string()); + } + } +} +#else +static void begin_loader_monitor() { +} + +static void end_loader_monitor(const fs::path &) { +} +#endif + +static std::set loaded_runtime_images(const fs::path & executable) { + std::set result; +#if defined(_WIN32) + const HANDLE snapshot = CreateToolhelp32Snapshot( + TH32CS_SNAPMODULE | TH32CS_SNAPMODULE32, + GetCurrentProcessId()); + if (snapshot == INVALID_HANDLE_VALUE) { + throw std::runtime_error("cannot enumerate loaded runtime modules"); + } + MODULEENTRY32W entry = {}; + entry.dwSize = sizeof(entry); + if (!Module32FirstW(snapshot, &entry)) { + CloseHandle(snapshot); + throw std::runtime_error("cannot read loaded runtime modules"); + } + do { + const fs::path reported_path = entry.szExePath; + const fs::path path = loaded_image_path(reported_path); + if (path != executable) { + result.insert(path); + } + } while (Module32NextW(snapshot, &entry)); + CloseHandle(snapshot); +#elif defined(__APPLE__) + const uint32_t count = _dyld_image_count(); + for (uint32_t index = 0; index < count; ++index) { + const char * name = _dyld_get_image_name(index); + if (name != nullptr && name[0] != '\0') { + const fs::path reported_path = name; + const fs::path path = loaded_image_path(reported_path); + if (path != executable) { + result.insert(path); + } + } + } +#elif defined(__linux__) + struct image_context { + const fs::path * executable; + std::set * result; + std::string error; + } context = {&executable, &result, {}}; + const auto callback = [](dl_phdr_info * info, size_t, void * data) { + image_context & context = *static_cast(data); + if (!context.error.empty() || info->dlpi_name == nullptr || info->dlpi_name[0] == '\0') { + return 0; + } + try { + const fs::path reported_path = info->dlpi_name; + if (!reported_path.is_absolute() && + (reported_path == "linux-vdso.so.1" || reported_path == "linux-gate.so.1")) { + return 0; + } + const fs::path path = loaded_image_path(reported_path); + if (path != *context.executable) { + context.result->insert(path); + } + } catch (const std::exception & error) { + context.error = error.what(); + } + return 0; + }; + if (dl_iterate_phdr(callback, &context) < 0 || !context.error.empty()) { + throw std::runtime_error( + context.error.empty() ? "cannot enumerate loaded runtime modules" : context.error); + } +#endif + return result; +} + +static bool is_lower_hex(const std::string & value, size_t length) { + return value.size() == length && std::all_of(value.begin(), value.end(), [](unsigned char character) { + return std::isdigit(character) || (character >= 'a' && character <= 'f'); + }); +} + +static fs::path runtime_library_directory(const fs::path & executable) { + const std::string profile = dsv41_runtime_receipt::profile; + if (profile == "co-located") { + return executable.parent_path(); + } + if (profile == "sibling-lib") { + return executable.parent_path().parent_path() / "lib"; + } + throw std::runtime_error("embedded runtime profile is invalid"); +} + +static void load_runtime_backends(const fs::path & executable) { +#if defined(GGML_BACKEND_DL) + const std::string directory = runtime_library_directory(executable).string(); + ggml_backend_load_all_from_path(directory.c_str()); +#else + (void) executable; + ggml_backend_load_all(); +#endif +} + +static void reject_loader_overrides() { + static const std::array names = { + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_IMAGE_SUFFIX", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "DYLD_ROOT_PATH", + "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", + "GGML_BACKEND_PATH", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + }; + for (const char * name : names) { + const char * value = std::getenv(name); + if (value != nullptr && value[0] != '\0') { + throw std::runtime_error(std::string("production trace execution forbids loader override: ") + name); + } + } +} + +static json runtime_libraries_json( + const fs::path & executable, + ggml_backend_dev_t selected_device, + const std::string & revision, + const std::string & selected_backend_component) { + if (selected_device == nullptr) { + throw std::runtime_error("cannot bind a null selected backend device"); + } + ggml_backend_reg_t selected_backend = ggml_backend_dev_backend_reg(selected_device); + if (selected_backend == nullptr) { + throw std::runtime_error("selected backend device has no runtime registry"); + } + if (dsv41_runtime_receipt::entries.size() != dsv41_runtime_receipt::components.size()) { + throw std::runtime_error("embedded runtime profile size differs from the receipt"); + } + + const fs::path library_directory = + canonical_path(runtime_library_directory(executable), "runtime library directory"); + std::map receipt_by_component; + std::map receipt_by_path; + std::set receipt_filenames; + std::set receipt_digests; + for (size_t index = 0; index < dsv41_runtime_receipt::entries.size(); ++index) { + const dsv41_runtime_receipt::entry & entry = dsv41_runtime_receipt::entries[index]; + const std::string component = entry.component; + const std::string filename = entry.filename; + const std::string digest = entry.sha256; + const std::string entry_revision = entry.revision; + if (component.empty() || filename.empty() || fs::path(filename).filename() != filename || + !is_lower_hex(digest, 64) || + !receipt_by_component.emplace(component, &entry).second || + !receipt_filenames.insert(filename).second || + !receipt_digests.insert(digest).second) { + throw std::runtime_error("embedded runtime receipt is not canonical and unique"); + } + if (component != dsv41_runtime_receipt::components[index]) { + throw std::runtime_error("embedded runtime profile differs from the receipt"); + } + const bool revision_bearing = component == "llama-common" || component == "ggml-base"; + if ((revision_bearing && entry_revision != revision) || + (!revision_bearing && !entry_revision.empty()) || + (!entry_revision.empty() && !is_lower_hex(entry_revision, 40))) { + throw std::runtime_error("embedded runtime receipt revision is invalid for " + component); + } + const fs::path expected_path = + canonical_path(library_directory / filename, "receipt runtime module"); + if (!receipt_by_path.emplace(expected_path, &entry).second) { + throw std::runtime_error("embedded runtime receipt path is duplicated"); + } + } + + const fs::path build_info_module = module_path(function_address(&llama_commit)); + const fs::path llama_module = module_path(function_address(&llama_model_load_from_file)); + const fs::path ggml_module = module_path(function_address(&ggml_init)); + const fs::path selected_backend_module = module_path(selected_backend); + const std::set images = loaded_runtime_images(executable); + std::set libraries; + const fs::path binary_directory = executable.parent_path(); + for (const fs::path & image : images) { + const bool in_runtime_root = + image.parent_path() == binary_directory || image.parent_path() == library_directory; + if (in_runtime_root || is_project_runtime_library(image)) { + libraries.insert(image); + } else if (!is_trusted_system_runtime_path(image)) { + throw std::runtime_error("loaded unclassified module outside trusted system roots: " + image.string()); + } + } + std::map loaded_by_component; + for (const fs::path & library : libraries) { + if (library.parent_path() != library_directory) { + throw std::runtime_error( + "loaded runtime module is outside the exporter runtime directory for the exact profile: " + + library.string()); + } + const auto receipt = receipt_by_path.find(library); + if (receipt == receipt_by_path.end()) { + throw std::runtime_error( + "loaded project runtime module is absent from the receipt: " + library.string()); + } + const std::string component = receipt->second->component; + if (!loaded_by_component.emplace(component, library).second) { + throw std::runtime_error("multiple loaded runtime modules map to receipt component " + component); + } + if (sha256_file(library) != receipt->second->sha256) { + throw std::runtime_error("loaded runtime module SHA-256 differs from the receipt: " + component); + } + } + if (loaded_by_component.size() != receipt_by_component.size()) { + for (const auto & item : receipt_by_component) { + if (loaded_by_component.count(item.first) == 0) { + throw std::runtime_error("receipt runtime component is not loaded: " + item.first); + } + } + throw std::runtime_error("loaded runtime component set differs from the receipt"); + } + + const std::array, 3> fixed_roles = {{ + {"llama-common", build_info_module}, + {"llama", llama_module}, + {"ggml-base", ggml_module}, + }}; + for (const auto & item : fixed_roles) { + const auto loaded = loaded_by_component.find(item.first); + if (loaded == loaded_by_component.end() || loaded->second != item.second) { + throw std::runtime_error( + "runtime symbol provider does not match receipt component " + std::string(item.first)); + } + } + const auto selected = loaded_by_component.find(selected_backend_component); + if (selected == loaded_by_component.end() || selected->second != selected_backend_module) { + throw std::runtime_error( + "selected backend module does not match receipt component " + selected_backend_component); + } + + json result = json::array(); + for (const fs::path & library : libraries) { + const dsv41_runtime_receipt::entry & receipt = *receipt_by_path.at(library); + std::string role = "runtime:" + std::string(receipt.component); + if (library == build_info_module) { + role = "build-info"; + } + if (library == llama_module) { + role = "llama"; + } + if (library == ggml_module) { + role = "ggml"; + } + if (library == selected_backend_module) { + role = "selected-backend"; + } + result.push_back({ + {"component", receipt.component}, + {"filename", receipt.filename}, + {"path", library.string()}, + {"sha256", receipt.sha256}, + {"role", std::move(role)}, + {"revision", receipt.revision[0] == '\0' ? json(nullptr) : json(receipt.revision)}, + }); + } + return result; +} + +static json runtime_build_json( + const fs::path & executable, + ggml_backend_dev_t selected_device, + const std::string & selected_backend_component, + char ** argv) { + const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked exporter"); + if (invoked_path != executable) { + throw std::runtime_error("invoked exporter path does not match the running executable"); + } + const std::string revision = BUILD_REVISION; + if (revision.size() != 40 || !std::all_of(revision.begin(), revision.end(), [](unsigned char value) { + return std::isdigit(value) || (value >= 'a' && value <= 'f'); + })) { + throw std::runtime_error("embedded exporter revision is invalid"); + } + const std::string linked_revision = llama_commit(); + const std::string ggml_revision = ggml_commit(); + if (linked_revision != revision || ggml_revision != revision) { + throw std::runtime_error( + "loaded runtime library revision differs from the exporter revision: llama=" + + linked_revision + ", ggml=" + ggml_revision + ", exporter=" + revision); + } +#if defined(DSV41_MANIFEST_TEST_HARNESS) + const std::string build_info = std::string(llama_build_info()) + " [test-only manifest harness]"; +#else + const std::string build_info = llama_build_info(); +#endif + return { + {"number", llama_build_number()}, + {"info", build_info}, + {"compiler", llama_compiler()}, + {"target", llama_build_target()}, + {"path", executable.string()}, + {"sha256", sha256_file(executable)}, + {"runtime_profile", { + {"name", dsv41_runtime_receipt::profile}, + {"components", dsv41_runtime_receipt::components}, + {"selected_backend_component", selected_backend_component}, + }}, + {"runtime_receipt_sha256", dsv41_runtime_receipt::sha256}, + {"runtime_libraries", runtime_libraries_json( + executable, selected_device, revision, selected_backend_component)}, + {"runtime_module_monitor", { +#if defined(__APPLE__) + {"mechanism", "dyld-add-image"}, +#else + {"mechanism", "pre-post-snapshot"}, +#endif + {"checked_after_trace", false}, + {"project_additions", json::array()}, + }}, + }; +} + +static std::string required_environment(const char * name) { + const char * value = std::getenv(name); + if (value == nullptr || value[0] == '\0') { + throw std::runtime_error(std::string("required environment variable is missing: ") + name); + } + return value; +} + +#if defined(__linux__) +static constexpr uint64_t DSV41_GIB = UINT64_C(1024)*1024*1024; + +static int required_descriptor(const char * name) { + const std::string value = required_environment(name); + size_t consumed = 0; + const long parsed = std::stol(value, &consumed); + if (consumed != value.size() || parsed <= STDERR_FILENO || + parsed > std::numeric_limits::max()) { + throw std::runtime_error(std::string("invalid descriptor environment: ") + name); + } + return static_cast(parsed); +} + +static std::string sha256_descriptor(int descriptor) { + sha256_t state; + sha256_init(&state); + std::vector buffer(8 * 1024 * 1024); + off_t offset = 0; + while (true) { + ssize_t count; + do { + count = pread(descriptor, buffer.data(), buffer.size(), offset); + } while (count < 0 && errno == EINTR); + if (count < 0) { + throw std::runtime_error("failed while hashing held model descriptor"); + } + if (count == 0) { + break; + } + sha256_update(&state, buffer.data(), static_cast(count)); + offset += count; + } + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_final(&state, digest); + return sha256_hex(digest); +} + +static json model_descriptor_identity(int descriptor, const fs::path & model_path, bool hash_bytes) { + struct stat status {}; + const int status_flags = fcntl(descriptor, F_GETFL); + const int descriptor_flags = fcntl(descriptor, F_GETFD); + if (fstat(descriptor, &status) != 0 || status_flags < 0 || descriptor_flags < 0) { + throw std::runtime_error("cannot inspect held model descriptor"); + } + if (!S_ISREG(status.st_mode) || status.st_nlink < 1 || + (status_flags & O_ACCMODE) != O_RDONLY || descriptor_flags != 0) { + throw std::runtime_error("held model descriptor policy is invalid"); + } + json result = { + {"format", "dsv41-model-file-identity"}, + {"version", 1}, + {"path", model_path.string()}, + {"device", static_cast(status.st_dev)}, + {"inode", static_cast(status.st_ino)}, + {"owner_uid", static_cast(status.st_uid)}, + {"owner_gid", static_cast(status.st_gid)}, + {"mode", static_cast(status.st_mode & 07777)}, + {"link_count", static_cast(status.st_nlink)}, + {"byte_count", static_cast(status.st_size)}, + {"modified_ns", + static_cast(status.st_mtim.tv_sec)*INT64_C(1000000000) + status.st_mtim.tv_nsec}, + {"changed_ns", + static_cast(status.st_ctim.tv_sec)*INT64_C(1000000000) + status.st_ctim.tv_nsec}, + {"status_flags", status_flags}, + {"source_descriptor_flags", FD_CLOEXEC}, + {"target_descriptor_flags", 0}, + }; + if (hash_bytes) { + result["sha256"] = sha256_descriptor(descriptor); + } + return result; +} + +static json validate_model_descriptor(const fs::path & model_path, bool hash_bytes) { + const int descriptor = required_descriptor("DSV41_MODEL_DESCRIPTOR"); + json expected; + try { + expected = json::parse(required_environment("DSV41_MODEL_DESCRIPTOR_IDENTITY")); + } catch (const json::exception & error) { + throw std::runtime_error(std::string("model descriptor identity JSON is invalid: ") + error.what()); + } + const json observed = model_descriptor_identity(descriptor, model_path, hash_bytes); + if (expected != observed) { + throw std::runtime_error("held model descriptor differs from the runner identity"); + } + return observed; +} + +static std::vector namespace_pid_chain() { + std::ifstream status("/proc/self/status"); + if (!status) { + throw std::runtime_error("cannot read namespace-local process status"); + } + std::string line; + while (std::getline(status, line)) { + if (line.rfind("NSpid:", 0) != 0) { + continue; + } + std::istringstream values(line.substr(6)); + std::vector result; + int64_t value = 0; + while (values >> value) { + result.push_back(value); + } + if (result.empty() || result.back() != getpid()) { + throw std::runtime_error("namespace-local NSpid identity is invalid"); + } + return result; + } + throw std::runtime_error("namespace-local NSpid identity is missing"); +} + +static void require_live_pidfd(int descriptor) { + if (fcntl(descriptor, F_GETFD) != 0) { + throw std::runtime_error("watchdog pidfd descriptor flags are invalid"); + } + std::array target {}; + const std::string descriptor_path = "/proc/self/fd/" + std::to_string(descriptor); + const ssize_t size = readlink(descriptor_path.c_str(), target.data(), target.size() - 1); + if (size <= 0 || std::string(target.data(), static_cast(size)) != "anon_inode:[pidfd]") { + throw std::runtime_error("watchdog authority descriptor is not a pidfd"); + } + pollfd observed {descriptor, POLLIN, 0}; + const int result = poll(&observed, 1, 0); + if (result < 0 || result != 0 || observed.revents != 0) { + throw std::runtime_error("watchdog authority pidfd is not live"); + } +} + +static json validate_watchdog(const json & data) { + const int64_t pid = data.value("watchdog_pid", INT64_C(0)); + const int64_t guardian_pid = data.value("guardian_pid", INT64_C(0)); + const int64_t child_pid = data.value("child_pid", INT64_C(0)); + const int64_t child_pgid = data.value("child_process_group_id", INT64_C(0)); + if (data.value("format", "") != "strix-memory-watchdog-lease" || data.value("version", 0) != 2) { + throw std::runtime_error("watchdog lease format is invalid"); + } + if (data.value("soft_bytes", UINT64_C(0)) != 116*DSV41_GIB || + data.value("emergency_bytes", UINT64_C(0)) != 118*DSV41_GIB || + data.value("strict_ceiling_bytes", UINT64_C(0)) != 120*DSV41_GIB || + data.value("grace_seconds", 0.0) != 30.0 || + data.value("sample_interval_seconds", 0.0) != 1.0 || + data.value("procfs_root", "") != "/proc") { + throw std::runtime_error("watchdog execution policy is invalid"); + } + if (pid <= 1 || guardian_pid <= 1 || child_pid <= 1 || child_pgid <= 1) { + throw std::runtime_error("watchdog host identity is invalid"); + } + const json authority = data.value("namespace_authority", json::object()); + const int authority_descriptor = required_descriptor("DSV41_WATCHDOG_PIDFD"); + if (authority.value("format", "") != "dsv41-watchdog-namespace-authority" || + authority.value("version", 0) != 1 || + authority.value("mechanism", "") != "inherited-pidfd" || + authority.value("descriptor", -1) != authority_descriptor || + authority.value("host_procfs_root", "") != "/proc" || + authority.value("watchdog_pid", INT64_C(0)) != pid || + authority.value("watchdog_start_time_ticks", UINT64_C(0)) != + data.value("watchdog_start_time_ticks", UINT64_C(0)) || + authority.value("watchdog_executable_path", "") != + data.value("watchdog_executable_path", "") || + authority.value("watchdog_command_sha256", "") != + data.value("watchdog_command_sha256", "") || + authority.value("guardian_pid", INT64_C(0)) != guardian_pid || + authority.value("child_pid", INT64_C(0)) != child_pid || + authority.value("child_process_group_id", INT64_C(0)) != child_pgid) { + throw std::runtime_error("watchdog namespace authority does not match the host audit"); + } + require_live_pidfd(authority_descriptor); + if (getpid() != 2 || getppid() != 1 || getpgrp() != getpid() || getsid(0) != getpid() || + canonical_path("/proc/self", "namespace-local process") != + canonical_path("/proc/" + std::to_string(getpid()), "namespace-local PID") || + !fs::is_directory("/proc/1")) { + throw std::runtime_error("trace exporter namespace-local process identity is invalid"); + } + const std::vector namespace_pids = namespace_pid_chain(); + const fs::path script_path = data.value("watchdog_script_path", ""); + if (script_path.empty() || data.value("watchdog_revision", "") != + "778db6f50eae04e6c232c69b9575bdbd0747962b" || + data.value("watchdog_script_sha256", "") != WATCHDOG_SCRIPT_SHA256 || + sha256_file(script_path) != WATCHDOG_SCRIPT_SHA256) { + throw std::runtime_error("watchdog script identity changed"); + } + const fs::path heartbeat_path = data.value("heartbeat_path", ""); + const double max_age = data.value("max_heartbeat_age_seconds", 0.0); + if (heartbeat_path.empty() || max_age <= 0 || max_age > 30) { + throw std::runtime_error("watchdog heartbeat configuration is invalid"); + } + if (fs::canonical(required_environment("STRIX_MEMORY_WATCHDOG_LEASE_PATH")) != + fs::canonical(fs::path(data.value("lease_path", ""))) || + fs::canonical(required_environment("STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH")) != + fs::canonical(heartbeat_path) || + fs::canonical(required_environment("STRIX_MEMORY_WATCHDOG_AUDIT_PATH")) != + fs::canonical(fs::path(data.value("audit_live_path", ""))) || + std::stod(required_environment("STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS")) != max_age) { + throw std::runtime_error("watchdog lease does not match the inherited environment"); + } + const json child_command = data.value("command", json::array()); + if (!child_command.is_array() || child_command.empty()) { + throw std::runtime_error("watchdog child command is invalid"); + } + for (const json & argument : child_command) { + if (!argument.is_string()) { + throw std::runtime_error("watchdog child command is invalid"); + } + } + const std::string child_command_json = child_command.dump(-1, ' ', true); + if (sha256_data( + reinterpret_cast(child_command_json.data()), + child_command_json.size()) != data.value("child_command_sha256", "")) { + throw std::runtime_error("watchdog child command SHA-256 is invalid"); + } + const std::vector heartbeat_bytes = read_file(heartbeat_path); + json heartbeat; + try { + heartbeat = json::parse(heartbeat_bytes.begin(), heartbeat_bytes.end()); + } catch (const json::exception & error) { + throw std::runtime_error(std::string("watchdog heartbeat is invalid: ") + error.what()); + } + if (heartbeat.value("format", "") != "strix-memory-watchdog-heartbeat" || + heartbeat.value("version", 0) != 2 || + heartbeat.value("lease_id", "") != data.value("lease_id", "") || + heartbeat.value("sequence", INT64_C(-1)) < 0 || + heartbeat.value("state", "") != "active" || + heartbeat.value("updated_at", "").empty() || + heartbeat.value("watchdog_pid", INT64_C(0)) != pid || + heartbeat.value("watchdog_start_time_ticks", UINT64_C(0)) != + data.value("watchdog_start_time_ticks", UINT64_C(0)) || + heartbeat.value("child_pid", INT64_C(0)) != child_pid || + heartbeat.value("child_process_group_id", INT64_C(0)) != child_pgid) { + throw std::runtime_error("watchdog heartbeat identity is invalid"); + } + const json heartbeat_sample = heartbeat.value("sample", json::object()); + const std::string audit_record_sha256 = heartbeat_sample.value("audit_record_sha256", ""); + if (audit_record_sha256.size() != 64) { + throw std::runtime_error("watchdog heartbeat audit identity is invalid"); + } + const uint64_t updated_monotonic_ns = heartbeat.value("updated_monotonic_ns", UINT64_C(0)); + struct timespec now; + if (updated_monotonic_ns == 0 || clock_gettime(CLOCK_MONOTONIC, &now) != 0) { + throw std::runtime_error("watchdog heartbeat monotonic timestamp is invalid"); + } + const uint64_t now_monotonic_ns = + static_cast(now.tv_sec)*UINT64_C(1000000000) + static_cast(now.tv_nsec); + const uint64_t max_age_ns = static_cast(max_age*1000000000.0); + if (updated_monotonic_ns > now_monotonic_ns || + now_monotonic_ns - updated_monotonic_ns > max_age_ns) { + throw std::runtime_error("watchdog heartbeat is stale"); + } + const fs::path audit_path = data.value("audit_live_path", ""); + if (audit_path.empty()) { + throw std::runtime_error("watchdog audit path is invalid"); + } + struct stat audit_stat; + const int audit_fd = data.value("audit_fd", -1); + if (audit_fd < 0 || lstat(audit_path.c_str(), &audit_stat) != 0 || + !S_ISREG(audit_stat.st_mode) || + static_cast(audit_stat.st_dev) != data.value("audit_device", UINT64_C(0)) || + static_cast(audit_stat.st_ino) != data.value("audit_inode", UINT64_C(0)) || + audit_stat.st_uid != getuid() || + (audit_stat.st_mode & 0777) != 0600 || + data.value("audit_mode", UINT64_C(0)) != 0600) { + throw std::runtime_error("watchdog persistent audit identity changed"); + } + const int local_audit_fd = open(audit_path.c_str(), O_RDONLY | O_NOFOLLOW); + if (local_audit_fd < 0) { + throw std::runtime_error("cannot open watchdog persistent audit"); + } + const int lock_result = flock(local_audit_fd, LOCK_EX | LOCK_NB); + if (lock_result == 0) { + flock(local_audit_fd, LOCK_UN); + close(local_audit_fd); + throw std::runtime_error("watchdog does not hold the persistent audit lock"); + } + if (errno != EWOULDBLOCK && errno != EAGAIN) { + close(local_audit_fd); + throw std::runtime_error("cannot inspect watchdog persistent audit lock"); + } + close(local_audit_fd); + std::ifstream audit_stream(audit_path); + if (!audit_stream) { + throw std::runtime_error("cannot read watchdog persistent audit"); + } + std::string audit_line; + bool found_audit_record = false; + while (std::getline(audit_stream, audit_line)) { + audit_line.push_back('\n'); + if (sha256_data(audit_line.data(), audit_line.size()) == audit_record_sha256) { + found_audit_record = true; + break; + } + } + if (!found_audit_record) { + throw std::runtime_error("watchdog heartbeat audit record is missing"); + } + require_live_pidfd(authority_descriptor); + return { + {"format", "dsv41-watchdog-namespace-binding"}, + {"version", 1}, + {"authority", "inherited-pidfd"}, + {"host_watchdog_pid", pid}, + {"host_watchdog_process_group_id", + authority.value("watchdog_process_group_id", INT64_C(0))}, + {"host_watchdog_start_time_ticks", + data.value("watchdog_start_time_ticks", UINT64_C(0))}, + {"local_pid", getpid()}, + {"local_parent_pid", getppid()}, + {"local_process_group_id", getpgrp()}, + {"local_session_id", getsid(0)}, + {"namespace_pids", namespace_pids}, + {"private_procfs", true}, + }; +} +#endif + +static void bind_memory_audit_metadata(json & result, const json & audit) { + result["accelerator"] = audit.value("accelerator", json::object()); + result["storage"] = audit.value("storage", json::object()); + result["storage_policy"] = audit.value("storage_policy", json::object()); +} + +static json audit_reference(const char * environment_name, const char * expected_kind) { + const fs::path path = required_environment(environment_name); + dsv41::require_nvme_path(path, "audit"); + const std::vector bytes = read_file(path); + json audit; + try { + audit = json::parse(bytes.begin(), bytes.end()); + } catch (const json::exception & error) { + throw std::runtime_error(std::string("invalid audit JSON: ") + error.what()); + } + if (audit.value("kind", "") != expected_kind) { + throw std::runtime_error(std::string("audit kind mismatch for ") + expected_kind); + } + if (audit.value("environment", json::object()).value("HIP_LAUNCH_BLOCKING", "") != "1") { + throw std::runtime_error(std::string("audit environment mismatch for ") + expected_kind); + } + const int64_t created = audit.value("created_unix", INT64_C(0)); + const int64_t now = static_cast(std::time(nullptr)); + if (created <= 0 || now < created || now - created > 300) { + throw std::runtime_error(std::string("audit is stale: ") + expected_kind); + } + if (std::string(expected_kind) == "swap" && audit["data"].value("enabled", true)) { + throw std::runtime_error("swap audit reports enabled swap"); + } + json result = { + {"path", fs::absolute(path).lexically_normal().string()}, + {"sha256", sha256_data(bytes.data(), bytes.size())}, + {"created_unix", created}, + }; +#if defined(__linux__) + if (std::string(expected_kind) == "watchdog") { + result["namespace_binding"] = validate_watchdog(audit["data"]); + } +#endif + if (std::string(expected_kind) == "watchdog") { + result["data"] = audit["data"]; + } else if (std::string(expected_kind) == "memory") { + bind_memory_audit_metadata(result, audit); + } + return result; +} + +static std::string tensor_dtype(const ggml_tensor * tensor) { + switch (tensor->type) { + case GGML_TYPE_F32: return "f32"; + case GGML_TYPE_BF16: return "bf16"; + case GGML_TYPE_I32: return "i32"; + case GGML_TYPE_I8: return "i8"; + default: throw std::runtime_error( + std::string("unsupported trace tensor type: ") + ggml_type_name(tensor->type)); + } +} + +static std::vector tensor_shape(const ggml_tensor * tensor) { + int rank = GGML_MAX_DIMS; + while (rank > 2 && tensor->ne[rank - 1] == 1) { + --rank; + } + std::vector result; + result.reserve(rank); + for (int i = 0; i < rank; ++i) { + result.push_back(tensor->ne[i]); + } + return result; +} + +static void write_manifest_file(const fs::path & path, const json & manifest); + +class trace_writer { +public: + trace_writer(fs::path root, json manifest) : + root(std::move(root)), + blobs(this->root / "blobs"), + manifest(std::move(manifest)) { + if (fs::exists(this->root) && !fs::is_empty(this->root)) { + throw std::runtime_error("trace output directory is not empty: " + this->root.string()); + } + fs::create_directories(blobs); + events.open(this->root / "events.jsonl", std::ios::binary | std::ios::trunc); + if (!events) { + throw std::runtime_error("cannot create events.jsonl"); + } + this->manifest["trace_format"] = "dsv41-trace"; + this->manifest["trace_version"] = TRACE_VERSION; + } + + void set_execution(std::string phase, int step, int64_t token_start, int64_t token_count) { + this->phase = std::move(phase); + this->step = step; + this->token_start = token_start; + this->token_count = token_count; + } + + void add( + const std::string & component, + int layer, + const std::string & dtype, + const std::vector & shape, + const void * data, + size_t size, + const char * semantic_id_space = nullptr) { + if (error_message.size() != 0) { + return; + } + try { + const std::string digest = sha256_data(data, size); + const fs::path blob = blobs / (digest + ".bin"); + if (!fs::exists(blob)) { + const fs::path temp = blob.string() + ".tmp"; + { + std::ofstream output(temp, std::ios::binary | std::ios::trunc); + if (!output || (size != 0 && !output.write(static_cast(data), size))) { + throw std::runtime_error("cannot write trace blob"); + } + } + fs::rename(temp, blob); + } + + json event = { + {"trace_version", TRACE_VERSION}, + {"component", component}, + {"phase", phase}, + {"step", step}, + {"token_start", token_start}, + {"token_count", token_count}, + {"layer", layer >= 0 ? json(layer) : json(nullptr)}, + {"dtype", dtype}, + {"shape", shape}, + {"byte_order", "little"}, + {"byte_count", size}, + {"sha256", digest}, + {"blob", "blobs/" + digest + ".bin"}, + }; + if (semantic_id_space != nullptr) { + event["semantic_id_space"] = semantic_id_space; + } + events << event.dump() << '\n'; + events.flush(); + if (!events) { + throw std::runtime_error("cannot append trace event"); + } + ++event_count; + } catch (const std::exception & error) { + error_message = error.what(); + } + } + + void add_tensor(const ggml_tensor * tensor) { + const std::string name = tensor->name; + const auto descriptor = dsv41_trace_parse_name(name); + if (!descriptor) { + return; + } + const size_t size = ggml_nbytes(tensor); + buffer.resize(size); + ggml_backend_tensor_get(tensor, buffer.data(), 0, size); + add(descriptor->component, descriptor->layer, tensor_dtype(tensor), tensor_shape(tensor), + buffer.data(), size, descriptor->semantic_id_space); + } + + bool has_error() const { + return !error_message.empty(); + } + + const std::string & error() const { + return error_message; + } + + void fail(const std::string & message) { + if (error_message.empty()) { + error_message = message; + } + } + + void bind_runtime_post(json runtime_libraries) { + if (manifest["build"]["runtime_libraries"] != runtime_libraries) { + throw std::runtime_error("loaded runtime component set changed during protected trace generation"); + } + manifest["build"]["runtime_libraries_post"] = std::move(runtime_libraries); + manifest["build"]["runtime_module_monitor"]["checked_after_trace"] = true; + } + + void finish() { + if (has_error()) { + throw std::runtime_error(error_message); + } + events.close(); + manifest["event_count"] = event_count; + write_manifest_file(root / "manifest.json", manifest); + } + +private: + fs::path root; + fs::path blobs; + std::ofstream events; + json manifest; + std::vector buffer; + std::string phase = "unknown"; + std::string error_message; + int step = 0; + int64_t token_start = 0; + int64_t token_count = 0; + uint64_t event_count = 0; +}; + +static bool trace_callback(ggml_tensor * tensor, bool ask, void * user_data) { + auto * writer = static_cast(user_data); + try { + if (ask) { + return dsv41_trace_select_name(tensor->name); + } + writer->add_tensor(tensor); + return !writer->has_error(); + } catch (const std::exception & error) { + writer->fail(error.what()); + std::fprintf(stderr, "trace callback failed: %s\n", error.what()); + return false; + } +} + +static std::vector copy_logits(llama_context * ctx, int32_t n_vocab) { + const float * logits = llama_get_logits_ith(ctx, -1); + if (logits == nullptr) { + throw std::runtime_error("runtime did not produce final-token logits"); + } + return std::vector(logits, logits + n_vocab); +} + +static int32_t greedy_token(const std::vector & logits) { + if (logits.empty()) { + throw std::runtime_error("cannot select from empty logits"); + } + return static_cast(std::max_element(logits.begin(), logits.end()) - logits.begin()); +} + +static void decode_tokens( + llama_context * ctx, + trace_writer & writer, + const std::vector & tokens, + int32_t n_ubatch) { + int64_t offset = 0; + while (offset < static_cast(tokens.size())) { + const int32_t count = static_cast(std::min(n_ubatch, tokens.size() - offset)); + llama_batch batch = llama_batch_init(count, 0, 1); + for (int32_t i = 0; i < count; ++i) { + const bool logits = offset + i + 1 == static_cast(tokens.size()); + common_batch_add(batch, tokens[offset + i], offset + i, {0}, logits); + } + writer.set_execution("prefill", 0, offset, count); + const int result = llama_decode(ctx, batch); + llama_batch_free(batch); + if (result != 0) { + throw std::runtime_error("prefill failed at token " + std::to_string(offset)); + } + if (writer.has_error()) { + throw std::runtime_error(writer.error()); + } + offset += count; + } +} + +static std::string model_architecture(const llama_model * model) { + std::array buffer = {}; + const int32_t count = llama_model_meta_val_str( + model, "general.architecture", buffer.data(), buffer.size()); + if (count < 0) { + throw std::runtime_error("model has no general.architecture metadata"); + } + return buffer.data(); +} + +static std::vector command_line(int argc, char ** argv) { + std::vector result; + result.reserve(argc); + for (int i = 0; i < argc; ++i) { + result.emplace_back(argv[i]); + } + return result; +} + +static std::string command_line_json(int argc, char ** argv) { + return json(command_line(argc, argv)).dump(); +} + +static bool flash_attention_enabled(enum llama_flash_attn_type value) { + return value == LLAMA_FLASH_ATTN_TYPE_ENABLED; +} + +static std::string runtime_system_info(const common_params & params) { +#if defined(_WIN32) + const std::string platform = "Windows"; +#else + struct utsname info = {}; + if (uname(&info) != 0) { + throw std::runtime_error("cannot query operating system identity"); + } + const std::string platform = + std::string(info.sysname) + " " + info.release + " " + info.machine; +#endif + return platform + "; " + common_params_get_system_info(params); +} + +static std::string runtime_platform_name() { +#if defined(_WIN32) + return "windows"; +#elif defined(__APPLE__) + return "darwin"; +#elif defined(__linux__) + return "linux"; +#else + return "unknown"; +#endif +} + +static json accelerator_json(const dsv41::accelerator_attestation & accelerator) { + return { + {"format", "dsv41-accelerator-attestation"}, + {"version", 2}, + {"runtime_kind", "strix-rocm"}, + {"platform", "linux"}, + {"backend", "ROCm"}, + {"backend_device", accelerator.backend_device}, + {"backend_description", accelerator.backend_description}, + {"pci_device_id", accelerator.pci_device_id}, + {"kfd_node", accelerator.kfd_node}, + {"gpu_id", accelerator.gpu_id}, + {"gfx_target_version", accelerator.gfx_target_version}, + {"architecture", accelerator.architecture}, + {"source", "linux-kfd-sysfs"}, + }; +} + +static json storage_json(const dsv41::storage_attestation & storage) { + return { + {"format", "dsv41-storage-attestation"}, + {"version", 2}, + {"runtime_kind", "strix-rocm"}, + {"platform", "linux"}, + {"storage_kind", "linux-nvme"}, + {"resolved_path", storage.resolved_path.string()}, + {"existing_path", storage.existing_path.string()}, + {"mount_point", storage.mount_point}, + {"filesystem_type", storage.filesystem_type}, + {"mount_source", storage.mount_source}, + {"device_number", storage.device_number}, + {"block_device_path", storage.block_device_path.string()}, + {"nvme_device", storage.nvme_device}, + {"rotational", false}, + {"source", "linux-mountinfo-sysfs"}, + }; +} + +static json storage_policy_json() { + return { + {"format", "dsv41-state-storage-policy"}, + {"version", 1}, + {"expert_cache", "memory-resident"}, + {"kv_cache", "memory-resident"}, + {"external_cache_paths", json::array()}, + {"external_state_paths", json::array()}, + }; +} + +struct native_manifest_evidence { + json accelerator; + json paths; + json config; +}; + +static json complete_manifest( + json input, + native_manifest_evidence evidence, + const fs::path & executable, + ggml_backend_dev_t selected_device, + const std::string & selected_backend_component, + const std::string & system_info, + int argc, + char ** argv) { + static const std::array required = { + "model", "prompt", "audits", "expected", "event_count", + }; + if (!input.is_object()) { + throw std::runtime_error("manifest writer input is not a JSON object"); + } + for (const char * key : required) { + if (!input.contains(key) && std::string(key) != "event_count") { + throw std::runtime_error(std::string("manifest writer input is missing ") + key); + } + } + for (const auto & item : input.items()) { + if (std::find_if(required.begin(), required.end(), [&](const char * key) { + return item.key() == key; + }) == required.end()) { + throw std::runtime_error("manifest writer input has unexpected field: " + item.key()); + } + } + json manifest = { + {"trace_format", "dsv41-trace"}, + {"trace_version", TRACE_VERSION}, + {"runtime", "llama.cpp"}, + {"revision", BUILD_REVISION}, + {"build", runtime_build_json(executable, selected_device, selected_backend_component, argv)}, + {"model", std::move(input["model"])}, + {"prompt", std::move(input["prompt"])}, + {"accelerator", std::move(evidence.accelerator)}, + {"paths", std::move(evidence.paths)}, + {"storage_policy", storage_policy_json()}, + {"config", std::move(evidence.config)}, + {"comparison", { + {"tokens", "exact"}, + {"engram_rows", "exact"}, + {"expert_ids", "exact-original-id-space"}, + {"expert_weights", "byte-identical-f32"}, + {"attention_candidates", "exact"}, + {"logits", "byte-identical-f32"}, + }}, + {"environment", { + {"system_info", system_info}, + {"command", command_line_json(argc, argv)}, + }}, + {"audits", std::move(input["audits"])}, + {"expected", std::move(input["expected"])}, + }; + if (input.contains("event_count")) { + manifest["event_count"] = std::move(input["event_count"]); + } + return manifest; +} + +static void write_manifest_file(const fs::path & path, const json & manifest) { + const fs::path temp = path.string() + ".tmp"; + { + std::ofstream stream(temp, std::ios::binary | std::ios::trunc); + stream << manifest.dump() << '\n'; + if (!stream) { + throw std::runtime_error("cannot write trace manifest"); + } + } + fs::rename(temp, path); +} + +#if defined(DSV41_MANIFEST_TEST_HARNESS) +static native_manifest_evidence test_manifest_evidence( + const fs::path & input_path, + const fs::path & output_path, + ggml_backend_dev_t device) { + if (device == nullptr || ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_CPU) { + throw std::runtime_error("manifest writer test requires a local CPU device"); + } + ggml_backend_dev_props properties = {}; + ggml_backend_dev_get_props(device, &properties); + ggml_backend_reg_t backend = ggml_backend_dev_backend_reg(device); + if (backend == nullptr) { + throw std::runtime_error("manifest writer test CPU has no runtime registry"); + } + const fs::path input = canonical_path(input_path, "manifest writer test input"); + const fs::path output_parent = + canonical_path(fs::absolute(output_path).parent_path(), "manifest writer test output directory"); + const fs::path output = output_parent / output_path.filename(); + const fs::path repository = canonical_path(DSV41_SOURCE_ROOT, "manifest writer source root"); + return { + { + {"format", "dsv41-native-test-accelerator"}, + {"version", 1}, + {"runtime_kind", "native-test"}, + {"platform", runtime_platform_name()}, + {"backend", ggml_backend_reg_name(backend)}, + {"backend_device", properties.name == nullptr ? "" : properties.name}, + {"backend_description", properties.description == nullptr ? "" : properties.description}, + {"device_type", "cpu"}, + {"source", "ggml-runtime"}, + {"test_only", true}, + }, + { + {"format", "dsv41-native-test-paths"}, + {"version", 1}, + {"input", input.string()}, + {"output", output.string()}, + {"repository", repository.string()}, + {"temporary_directory", output_parent.string()}, + {"test_only", true}, + }, + { + {"format", "dsv41-native-test-config"}, + {"version", 1}, + {"runtime_kind", "native-test"}, + {"platform", runtime_platform_name()}, + {"device", properties.name == nullptr ? "" : properties.name}, + {"device_description", properties.description == nullptr ? "" : properties.description}, + {"device_type", "cpu"}, + {"build_target", llama_build_target()}, + {"flash_attention", false}, + {"gpu_layers", 0}, + {"test_only", true}, + }, + }; +} + +static void write_manifest_probe( + const fs::path & input_path, + const fs::path & output_path, + int argc, + char ** argv) { + const std::vector bytes = read_file(input_path); + json input = json::parse(bytes.begin(), bytes.end()); + static const std::array allowed = { + "model", "prompt", "audits", "expected", "event_count", + }; + if (!input.is_object()) { + throw std::runtime_error("manifest writer test input is not a JSON object"); + } + for (const char * key : allowed) { + if (!input.contains(key)) { + throw std::runtime_error(std::string("manifest writer test input is missing ") + key); + } + } + for (const auto & item : input.items()) { + if (std::find_if(allowed.begin(), allowed.end(), [&](const char * key) { + return item.key() == key; + }) == allowed.end()) { + throw std::runtime_error("manifest writer test input has unexpected field: " + item.key()); + } + } + common_init(); + const fs::path executable = current_executable_path(); + load_runtime_backends(executable); + ggml_backend_dev_t device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + json manifest = complete_manifest( + std::move(input), + test_manifest_evidence(input_path, output_path, device), + executable, + device, + "ggml-cpu", + runtime_platform_name() + " local CPU model-free manifest writer test", + argc, + argv); + begin_loader_monitor(); + end_loader_monitor(executable); + manifest["build"]["runtime_libraries_post"] = runtime_libraries_json( + executable, device, BUILD_REVISION, "ggml-cpu"); + if (manifest["build"]["runtime_libraries"] != manifest["build"]["runtime_libraries_post"]) { + throw std::runtime_error("loaded runtime component set changed during manifest writer test"); + } + manifest["build"]["runtime_module_monitor"]["checked_after_trace"] = true; + write_manifest_file(output_path, manifest); +} + +int main(int argc, char ** argv) { + std::setlocale(LC_NUMERIC, "C"); + try { + if (argc != 4 || std::string(argv[1]) != "--write-test-manifest") { + throw std::runtime_error("test manifest writer requires input and output paths"); + } + write_manifest_probe(argv[2], argv[3], argc, argv); + return 0; + } catch (const std::exception & error) { + std::fprintf(stderr, "test-deepseek-v41-manifest: %s\n", error.what()); + return 1; + } +} +#else +int main(int argc, char ** argv) { + std::setlocale(LC_NUMERIC, "C"); + try { + reject_loader_overrides(); +#if defined(__linux__) + if (argc == 3 && std::string(argv[1]) == "--dsv41-test-watchdog") { + const std::vector bytes = read_file(argv[2]); + std::cout << validate_watchdog(json::parse(bytes.begin(), bytes.end())).dump() << '\n'; + return 0; + } + if (argc == 3 && std::string(argv[1]) == "--dsv41-test-model-descriptor") { + const fs::path model_path = canonical_path(argv[2], "test model"); + std::cout << validate_model_descriptor(model_path, true).dump() << '\n'; + return 0; + } +#endif + if (argc == 2 && std::string(argv[1]) == "--version") { + common_init(); + const fs::path executable = current_executable_path(); + load_runtime_backends(executable); + const json build = runtime_build_json( + executable, + ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU), + "ggml-cpu", + argv); + std::cout << "version: deepseek-v41-trace (build " << build["number"] + << ", commit " << BUILD_REVISION << ")\n"; + std::cout << "built with " << build["compiler"].get() + << " for " << build["target"].get() << '\n'; + return 0; + } + if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-build") { + if (std::string(argv[2]) != "ROCm0") { + throw std::runtime_error("build attestation requires selected execution device ROCm0"); + } + common_init(); + const fs::path executable = current_executable_path(); + load_runtime_backends(executable); + ggml_backend_dev_t device = ggml_backend_dev_by_name(argv[2]); + if (device == nullptr) { + throw std::runtime_error("cannot find selected execution device ROCm0"); + } + std::cout << runtime_build_json(executable, device, "ggml-hip", argv).dump() << '\n'; + return 0; + } + if (argc == 3 && std::string(argv[1]) == "--dsv41-attest-device") { + common_init(); + load_runtime_backends(current_executable_path()); + const dsv41::accelerator_attestation accelerator = + dsv41::require_gfx1151_device(ggml_backend_dev_by_name(argv[2])); + std::cout << accelerator_json(accelerator).dump() << '\n'; + return 0; + } + + const uint16_t endian = 1; + if (*reinterpret_cast(&endian) != 1) { + throw std::runtime_error("trace writer requires a little-endian host"); + } + + common_params params; + params.escape = false; + params.warmup = false; + params.ctx_shift = false; + common_init(); + load_runtime_backends(current_executable_path()); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_RESULTS)) { + return 1; + } + if (params.model.path.empty() || params.prompt_file.empty() || params.out_file.empty()) { + throw std::runtime_error("-m, -bf, and -o are required"); + } + if (params.n_predict < 1) { + throw std::runtime_error("-n must request at least one deterministic decode step"); + } + const fs::path executable_path = current_executable_path(); + + const dsv41::storage_attestation model_storage = + dsv41::require_nvme_path(params.model.path, "model"); + const dsv41::storage_attestation prompt_storage = + dsv41::require_nvme_path(params.prompt_file, "prompt"); + const dsv41::storage_attestation output_storage = + dsv41::require_nvme_path(params.out_file, "trace output"); + const fs::path temporary_directory = required_environment("TMPDIR"); + dsv41::require_usable_directory(temporary_directory, "TMPDIR"); + const dsv41::storage_attestation temporary_storage = + dsv41::require_nvme_path(temporary_directory, "temporary directory"); + if (required_environment("HIP_LAUNCH_BLOCKING") != "1") { + throw std::runtime_error("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs"); + } + const json tokenizer_policy = json::parse(required_environment("DSV41_TOKENIZER_POLICY")); + if (!tokenizer_policy.is_object() || tokenizer_policy.size() != 5 || + !tokenizer_policy.contains("add_bos") || + !tokenizer_policy.contains("parse_special") || + !tokenizer_policy.contains("detokenize_special") || + !tokenizer_policy.contains("remove_leading_bos_before_detokenize") || + !tokenizer_policy.contains("require_round_trip") || + !tokenizer_policy["add_bos"].is_boolean() || + !tokenizer_policy["parse_special"].is_boolean() || + !tokenizer_policy["detokenize_special"].is_boolean() || + !tokenizer_policy["remove_leading_bos_before_detokenize"].is_boolean() || + !tokenizer_policy["require_round_trip"].is_boolean() || + !tokenizer_policy["parse_special"].get() || + !tokenizer_policy["detokenize_special"].get() || + !tokenizer_policy["require_round_trip"].get() || + tokenizer_policy["remove_leading_bos_before_detokenize"].get() != + tokenizer_policy["add_bos"].get()) { + throw std::runtime_error("DSV41_TOKENIZER_POLICY is not the exact approved tokenizer policy"); + } + if (params.devices.size() != 1 || params.devices[0] == nullptr) { + throw std::runtime_error("trace tool requires exactly one selected execution device"); + } + const dsv41::accelerator_attestation configured_accelerator = + dsv41::require_gfx1151_device(params.devices[0]); + const json memory_audit = audit_reference("DSV41_TRACE_MEMORY_AUDIT", "memory"); + if (memory_audit.value("accelerator", json::object()) != accelerator_json(configured_accelerator)) { + throw std::runtime_error("preflight accelerator audit does not match the selected execution device"); + } + if (memory_audit.value("storage_policy", json::object()) != storage_policy_json()) { + throw std::runtime_error("preflight external cache/state storage policy is invalid"); + } + const json audited_storage = memory_audit.value("storage", json::object()); + if (audited_storage.value("model", json::object()) != storage_json(model_storage) || + audited_storage.value("prompt", json::object()) != storage_json(prompt_storage) || + audited_storage.value("output", json::object()) != storage_json(output_storage) || + audited_storage.value("temporary_directory", json::object()) != storage_json(temporary_storage)) { + throw std::runtime_error("preflight storage audit does not match the selected execution paths"); + } + const json swap_audit = audit_reference("DSV41_TRACE_SWAP_AUDIT", "swap"); + const json watchdog_audit = audit_reference("DSV41_TRACE_WATCHDOG_AUDIT", "watchdog"); + + const std::vector prompt_bytes = read_file(params.prompt_file); + if (params.prompt.size() != prompt_bytes.size() || + !std::equal(prompt_bytes.begin(), prompt_bytes.end(), params.prompt.begin())) { + throw std::runtime_error("parsed prompt differs from exact prompt file bytes"); + } + + const fs::path model_path = model_storage.resolved_path; + const fs::path prompt_path = prompt_storage.resolved_path; + const fs::path output_path = output_storage.resolved_path; +#if !defined(__linux__) + throw std::runtime_error("trace model descriptor binding requires Linux"); +#else + const int model_descriptor = required_descriptor("DSV41_MODEL_DESCRIPTOR"); + const json model_file_identity = validate_model_descriptor(model_path, true); + params.model.path = "/proc/self/fd/" + std::to_string(model_descriptor); +#endif + llama_backend_init(); + llama_numa_init(params.numa); + common_init_result_ptr init = common_init_from_params(params); + llama_model * model = init->model(); + llama_context * ctx = init->context(); + if (model == nullptr || ctx == nullptr) { + throw std::runtime_error("failed to initialize llama.cpp"); + } + if (model_architecture(model) != "deepseek41") { + throw std::runtime_error("trace tool requires general.architecture=deepseek41"); + } + std::vector model_devices; + for (int32_t index = 0; index < llama_model_n_devices(model); ++index) { + model_devices.emplace_back(ggml_backend_dev_name(llama_model_get_device(model, index))); + } + if (model_devices != std::vector{"ROCm0"}) { + throw std::runtime_error("trace tool requires the loaded model to use only ROCm0"); + } + const dsv41::accelerator_attestation accelerator = + dsv41::require_gfx1151_device(llama_model_get_device(model, 0)); + if (accelerator_json(accelerator) != accelerator_json(configured_accelerator)) { + throw std::runtime_error("loaded model device differs from the pre-allocation accelerator attestation"); + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool add_bos = llama_vocab_get_add_bos(vocab); + if (add_bos != tokenizer_policy["add_bos"].get()) { + throw std::runtime_error("model tokenizer add_bos differs from the approved policy"); + } + const std::vector tokens = common_tokenize( + ctx, params.prompt, add_bos, tokenizer_policy["parse_special"].get()); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); + if (tokens.empty()) { + throw std::runtime_error("prompt tokenization produced no tokens"); + } + if (tokens.size() > llama_n_ctx(ctx)) { + throw std::runtime_error("prompt token count exceeds the configured context"); + } + if (tokens.size() + static_cast(params.n_predict) > llama_n_ctx(ctx)) { + throw std::runtime_error("prompt plus decode steps exceed the configured context"); + } + + std::vector all_layers(40); + for (int32_t layer = 0; layer < 40; ++layer) { + all_layers[layer] = layer; + } + + native_manifest_evidence evidence = { + accelerator_json(accelerator), + { + {"model", model_path.string()}, + {"prompt", prompt_path.string()}, + {"output", output_path.string()}, + {"repository", audited_storage["repository"].value("resolved_path", "")}, + {"temporary_directory", temporary_storage.resolved_path.string()}, + }, + { + {"context", llama_n_ctx(ctx)}, + {"batch", params.n_batch}, + {"ubatch", params.n_ubatch}, + {"device", model_devices[0]}, + {"device_architecture", accelerator.architecture}, + {"device_pci_id", accelerator.pci_device_id}, + {"decode_steps", params.n_predict}, + {"kv_type_k", ggml_type_name(params.cache_type_k)}, + {"kv_type_v", ggml_type_name(params.cache_type_v)}, + {"flash_attention", flash_attention_enabled(params.flash_attn_type)}, + {"gpu_layers", params.n_gpu_layers}, + {"load_mode", static_cast(params.load_mode)}, + {"expert_cache_slots", params.expert_cache_slots}, + {"expert_cache_bytes", static_cast(params.expert_cache_mib) << 20}, + {"tokenizer", tokenizer_policy}, +#if defined(__linux__) + {"model_file_identity", model_file_identity}, + {"watchdog_namespace", watchdog_audit["namespace_binding"]}, +#endif + {"deepseek41", { + {"layer_count", 40}, + {"vocab_size", n_vocab}, + {"engram_layers", {1, 14}}, + {"engram_rows_per_token", 24}, + {"expert_count", 384}, + {"experts_used", 6}, + {"candidate_source_layer", 20}, + {"candidate_topk_blocks", 2048}, + {"candidate_block_size", 8}, + {"index_top_k", 512}, + {"raw_attention_layers", {0, 1}}, + {"raw_attention_width", 128}, + {"candidate_propagation_layers", {24, 28, 32, 36}}, + }}, + }, + }; + json manifest_input = { + {"model", { + {"path", model_path.string()}, + {"architecture", "deepseek41"}, +#if defined(__linux__) + {"byte_count", model_file_identity["byte_count"]}, + {"sha256", model_file_identity["sha256"]}, +#else + {"byte_count", fs::file_size(model_path)}, + {"sha256", sha256_file(model_path)}, +#endif + }}, + {"prompt", { + {"path", prompt_path.string()}, + {"byte_count", prompt_bytes.size()}, + {"sha256", sha256_data(prompt_bytes.data(), prompt_bytes.size())}, + }}, + {"audits", { + {"memory", memory_audit}, + {"swap", swap_audit}, + {"watchdog", watchdog_audit}, + }}, + {"expected", { + {"prompt_tokens", tokens.size()}, + {"decode_steps", params.n_predict}, + {"components", { + {"prompt.bytes", {{"layers", nullptr}, {"input", "tokens"}}}, + {"prompt.tokens", {{"layers", nullptr}, {"input", "tokens"}}}, + {"engram.row_ids", {{"layers", {1, 14}}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"expert.ids", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"expert.weights", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"attn.source", {{"layers", all_layers}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"attn.candidate_blocks", {{"layers", {20}}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"attn.candidates", {{"layers", {24, 28, 32, 36}}, {"prefill", "tokens"}, {"decode", "steps"}}}, + {"logits.prefill", {{"layers", nullptr}, {"prefill", "final"}}}, + {"logits.decode", {{"layers", nullptr}, {"decode", "steps"}}}, + {"decode.greedy_token", {{"layers", nullptr}, {"decode", "steps"}}}, + }}, + }}, + }; + json manifest = complete_manifest( + std::move(manifest_input), + std::move(evidence), + executable_path, + params.devices[0], + "ggml-hip", + runtime_system_info(params), + argc, + argv); + + begin_loader_monitor(); + trace_writer writer(output_path, std::move(manifest)); + llama_set_eval_callback(ctx, trace_callback, &writer); + + writer.set_execution("input", 0, 0, tokens.size()); + writer.add("prompt.bytes", -1, "bytes", {static_cast(prompt_bytes.size())}, + prompt_bytes.data(), prompt_bytes.size()); + writer.add("prompt.tokens", -1, "i32", {static_cast(tokens.size())}, + tokens.data(), tokens.size()*sizeof(tokens[0])); + + decode_tokens(ctx, writer, tokens, params.n_ubatch); + std::vector logits = copy_logits(ctx, n_vocab); + writer.set_execution("prefill", 0, tokens.size() - 1, 1); + writer.add("logits.prefill", -1, "f32", {n_vocab}, logits.data(), logits.size()*sizeof(float)); + + int64_t position = tokens.size(); + for (int32_t step = 0; step < params.n_predict; ++step) { + const llama_token token = greedy_token(logits); + writer.set_execution("decode", step, position, 1); + writer.add("decode.greedy_token", -1, "i32", {1}, &token, sizeof(token)); + + llama_batch batch = llama_batch_init(1, 0, 1); + common_batch_add(batch, token, position, {0}, true); + const int result = llama_decode(ctx, batch); + llama_batch_free(batch); + if (result != 0) { + throw std::runtime_error("decode failed at step " + std::to_string(step)); + } + if (writer.has_error()) { + throw std::runtime_error(writer.error()); + } + logits = copy_logits(ctx, n_vocab); + writer.add("logits.decode", -1, "f32", {n_vocab}, logits.data(), logits.size()*sizeof(float)); + ++position; + } + +#if defined(__linux__) + if (validate_watchdog(watchdog_audit["data"]) != watchdog_audit["namespace_binding"]) { + throw std::runtime_error("watchdog namespace binding changed during trace execution"); + } + if (validate_model_descriptor(model_path, true) != model_file_identity) { + throw std::runtime_error("held model descriptor changed during trace execution"); + } +#endif + end_loader_monitor(executable_path); + writer.bind_runtime_post(runtime_libraries_json( + executable_path, params.devices[0], BUILD_REVISION, "ggml-hip")); + writer.finish(); + llama_backend_free(); + return 0; + } catch (const std::exception & error) { + std::fprintf(stderr, "llama-deepseek-v41-trace: %s\n", error.what()); + return 1; + } +} +#endif diff --git a/tools/deepseek-v41-trace/preflight.py b/tools/deepseek-v41-trace/preflight.py new file mode 100644 index 000000000000..e729102f39bf --- /dev/null +++ b/tools/deepseek-v41-trace/preflight.py @@ -0,0 +1,1590 @@ +#!/usr/bin/env python3 + +import json +import hashlib +import importlib.util +import os +import platform +import plistlib +import re +import stat +import subprocess +import sys +import time +from datetime import datetime +from pathlib import Path +from typing import Callable + +from trace_format import ( + NO_EXTERNAL_STATE_STORAGE, + TraceError, + install_trust_sha256, + runtime_build_evidence_sha256, + validate_install_trust_evidence, + validate_runtime_build_evidence, + validate_watchdog_event, +) + +FORBIDDEN_ROOT = Path("/mnt/bigspace") +SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 +WATCHDOG_EMERGENCY_LIMIT = 118 * 1024 * 1024 * 1024 +STRICT_MEMORY_LIMIT = 120 * 1024 * 1024 * 1024 +MAX_WATCHDOG_HEARTBEAT_AGE = 30.0 +WATCHDOG_LEASE_FORMAT = "strix-memory-watchdog-lease" +WATCHDOG_HEARTBEAT_FORMAT = "strix-memory-watchdog-heartbeat" +WATCHDOG_VERSION = 2 +WATCHDOG_STARTUP_TIMEOUT_SECONDS = 5.0 +WATCHDOG_LEASE_ENV = "STRIX_MEMORY_WATCHDOG_LEASE_PATH" +WATCHDOG_HEARTBEAT_ENV = "STRIX_MEMORY_WATCHDOG_HEARTBEAT_PATH" +WATCHDOG_AUDIT_ENV = "STRIX_MEMORY_WATCHDOG_AUDIT_PATH" +WATCHDOG_MAX_AGE_ENV = "STRIX_MEMORY_WATCHDOG_HEARTBEAT_MAX_AGE_SECONDS" +WATCHDOG_REVISION = "778db6f50eae04e6c232c69b9575bdbd0747962b" +WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f" +APPROVED_WATCHDOGS = {WATCHDOG_SCRIPT_SHA256: WATCHDOG_REVISION} +_WATCHDOG_GUARD = None + + +class PreflightError(RuntimeError): + pass + + +def strict_json_loads(data: str) -> object: + def reject_duplicates(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise PreflightError(f"duplicate JSON key: {key}") + result[key] = value + return result + + try: + return json.loads(data, object_pairs_hook=reject_duplicates) + except json.JSONDecodeError as error: + raise PreflightError(f"invalid JSON: {error}") from error + + +def resolved(path: Path) -> Path: + return path.expanduser().resolve() + + +def require_no_symlink_components(path: Path, label: str) -> Path: + absolute = path.expanduser().absolute() + current = Path(absolute.anchor) + for part in absolute.parts[1:]: + current /= part + if current.is_symlink(): + raise PreflightError(f"{label} must not be a symlink or contain symlink components") + if not current.exists(): + break + return absolute + + +def reject_forbidden_path( + path: Path, + label: str, + forbidden_root: Path = FORBIDDEN_ROOT) -> Path: + absolute = path.expanduser().absolute() + try: + absolute.relative_to(forbidden_root) + except ValueError: + return absolute + raise PreflightError(f"{label} must not use /mnt/bigspace: {absolute}") + + +def require_safe_tmpdir_path(path: Path) -> Path: + if not path.is_absolute(): + raise PreflightError("TMPDIR must use an absolute literal path") + lexical_path = reject_forbidden_path(path, "TMPDIR") + try: + status = os.lstat(path) + except OSError as error: + raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") from error + if stat.S_ISLNK(status.st_mode): + raise PreflightError("TMPDIR must not be a symlink or contain symlink components") + if not stat.S_ISDIR(status.st_mode): + raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") + if not os.access(path, os.W_OK | os.X_OK): + raise PreflightError("TMPDIR must be an existing writable directory at its original lexical path") + require_no_symlink_components(lexical_path, "TMPDIR") + return lexical_path + + +def _decode_mount_field(value: str) -> str: + return re.sub( + r"\\([0-7]{3})", + lambda match: chr(int(match.group(1), 8)), + value, + ) + + +def _existing_ancestor(path: Path) -> Path: + current = path + while not current.exists(): + if current == current.parent: + raise PreflightError(f"path has no existing parent: {path}") + current = current.parent + return current.resolve(strict=True) + + +def storage_attestation( + path: Path, + label: str, + *, + mountinfo_path: Path = Path("/proc/self/mountinfo"), + sys_dev_block_root: Path = Path("/sys/dev/block"), + sys_class_block_root: Path = Path("/sys/class/block"), + forbidden_root: Path = FORBIDDEN_ROOT) -> dict[str, object]: + lexical_path = reject_forbidden_path(path, label, forbidden_root) + path = resolved(lexical_path) + try: + path.relative_to(forbidden_root) + except ValueError: + pass + else: + raise PreflightError(f"{label} must not use /mnt/bigspace: {path}") + + existing = _existing_ancestor(path) + mounts: list[tuple[Path, str, str, str]] = [] + for line in read_proc_lines(mountinfo_path): + fields = line.split() + try: + separator = fields.index("-") + mount = ( + Path(_decode_mount_field(fields[4])), + fields[separator + 1], + _decode_mount_field(fields[separator + 2]), + fields[2], + ) + except (IndexError, ValueError) as error: + raise PreflightError(f"invalid mountinfo record: {line}") from error + try: + existing.relative_to(mount[0]) + except ValueError: + continue + mounts.append(mount) + if not mounts: + raise PreflightError(f"{label} mount cannot be resolved: {path}") + mount_point, filesystem_type, mount_source, device_number = max( + mounts, key=lambda item: len(item[0].parts)) + if device_number.split(":", 1)[0] == "0": + source_device = mount_source.split("[", 1)[0] + source_name = Path(source_device).name + source_dev_path = sys_class_block_root / source_name / "dev" + if not source_device.startswith("/dev/") or not source_dev_path.is_file(): + raise PreflightError(f"{label} is not backed by a local block device: {path}") + try: + device_number = source_dev_path.read_text(encoding="ascii").strip() + except OSError as error: + raise PreflightError(f"{label} backing device identity cannot be read: {error}") from error + if re.fullmatch(r"[0-9]+:[0-9]+", device_number) is None: + raise PreflightError(f"{label} backing device identity is invalid: {device_number}") + + device_link = sys_dev_block_root / device_number + if not device_link.is_symlink(): + raise PreflightError(f"{label} block device cannot be resolved: {path}") + try: + block_device = device_link.resolve(strict=True) + except OSError as error: + raise PreflightError(f"{label} block device cannot be resolved: {error}") from error + rotational_path = None + for candidate in (block_device, *block_device.parents): + path_candidate = candidate / "queue" / "rotational" + if path_candidate.is_file(): + rotational_path = path_candidate + break + if rotational_path is None: + raise PreflightError(f"{label} block device rotational state cannot be resolved: {block_device}") + try: + rotational = rotational_path.read_text(encoding="ascii").strip() + except OSError as error: + raise PreflightError(f"{label} block device rotational state cannot be read: {error}") from error + if rotational != "0": + raise PreflightError(f"{label} must use non-rotational storage: {path}") + + nvme_device = next( + (part for part in reversed(block_device.parts) if re.fullmatch(r"nvme[0-9]+(?:c[0-9]+)?n[0-9]+", part)), + None, + ) + if nvme_device is None: + raise PreflightError(f"{label} must use an NVMe block device: {block_device}") + return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "storage_kind": "linux-nvme", + "resolved_path": str(path), + "existing_path": str(existing), + "mount_point": str(mount_point), + "filesystem_type": filesystem_type, + "mount_source": mount_source, + "device_number": device_number, + "block_device_path": str(block_device), + "nvme_device": nvme_device, + "rotational": False, + "source": "linux-mountinfo-sysfs", + } + + +def require_nvme_path( + path: Path, + label: str, + *, + mountinfo_path: Path = Path("/proc/self/mountinfo"), + sys_dev_block_root: Path = Path("/sys/dev/block"), + sys_class_block_root: Path = Path("/sys/class/block")) -> Path: + attestation = storage_attestation( + path, + label, + mountinfo_path=mountinfo_path, + sys_dev_block_root=sys_dev_block_root, + sys_class_block_root=sys_class_block_root, + ) + return _attested_resolved_path(attestation, label) + + +def _diskutil_info(path: Path) -> dict[str, object]: + try: + df = subprocess.check_output( + ["df", "-P", str(path)], + text=True, + stderr=subprocess.STDOUT, + ).splitlines() + if len(df) < 2: + raise PreflightError(f"df cannot resolve a mounted volume for {path}") + fields = df[-1].split(maxsplit=5) + if len(fields) != 6 or not fields[5].startswith("/"): + raise PreflightError(f"df returned invalid mount evidence for {path}") + mount_point = fields[5] + data = subprocess.check_output( + ["diskutil", "info", "-plist", mount_point], + stderr=subprocess.STDOUT, + ) + record = plistlib.loads(data) + except (OSError, subprocess.CalledProcessError, plistlib.InvalidFileException) as error: + raise PreflightError(f"diskutil cannot attest storage for {path}: {error}") from error + if not isinstance(record, dict): + raise PreflightError(f"diskutil returned invalid storage evidence for {path}") + record["_dsv41_mount_point"] = mount_point + return record + + +def darwin_storage_attestation( + path: Path, + label: str, + *, + disk_info: Callable[[Path], dict[str, object]] = _diskutil_info, + forbidden_root: Path = FORBIDDEN_ROOT) -> dict[str, object]: + lexical_path = reject_forbidden_path(path, label, forbidden_root) + path = resolved(lexical_path) + try: + path.relative_to(forbidden_root) + except ValueError: + pass + else: + raise PreflightError(f"{label} must not use /mnt/bigspace: {path}") + existing = _existing_ancestor(path) + record = dict(disk_info(existing)) + mount_point = record.pop("_dsv41_mount_point", record.get("MountPoint")) + filesystem_type = record.get("FilesystemType") + device_identifier = record.get("DeviceIdentifier") + parent_whole_disk = record.get("ParentWholeDisk") + bus_protocol = record.get("BusProtocol") + if record.get("Internal") is not True or record.get("SolidState") is not True: + raise PreflightError(f"{label} must use internal non-rotational storage: {path}") + if record.get("VolumeNetwork") is True or record.get("DiskImage") is True: + raise PreflightError(f"{label} must use local storage: {path}") + for name, value in ( + ("mount point", mount_point), + ("filesystem type", filesystem_type), + ("device identifier", device_identifier), + ("parent whole disk", parent_whole_disk), + ("bus protocol", bus_protocol)): + if not isinstance(value, str) or not value: + raise PreflightError(f"{label} {name} cannot be resolved: {path}") + if not mount_point.startswith("/"): + raise PreflightError(f"{label} mount point is invalid: {mount_point}") + if bus_protocol.lower() not in {"nvme", "apple fabric"}: + raise PreflightError(f"{label} storage is not NVMe-backed: {bus_protocol}") + try: + filesystem_device = os.stat(existing).st_dev + if filesystem_device != os.stat(mount_point).st_dev: + raise PreflightError(f"{label} filesystem identity differs from its attested mount: {path}") + except OSError as error: + raise PreflightError(f"{label} filesystem identity cannot be read: {error}") from error + return { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "storage_kind": "darwin-local-solid-state", + "resolved_path": str(path), + "existing_path": str(existing), + "mount_point": mount_point, + "filesystem_type": filesystem_type, + "device_identifier": device_identifier, + "parent_whole_disk": parent_whole_disk, + "bus_protocol": bus_protocol, + "filesystem_device": filesystem_device, + "internal": True, + "solid_state": True, + "source": "diskutil-info-plist", + } + + +def _attested_resolved_path(attestation: dict[str, object], label: str) -> Path: + resolved_path = attestation["resolved_path"] + if not isinstance(resolved_path, str): + raise PreflightError(f"{label} resolved path evidence is invalid") + return Path(resolved_path) + + +def safe_trace_path(root: Path, relative: Path | str) -> Path: + root = root.expanduser().absolute() + if root.is_symlink(): + raise PreflightError("trace output root must not be a symlink") + root = root.resolve() + relative = Path(relative) + if relative.is_absolute() or ".." in relative.parts: + raise PreflightError(f"trace output path is outside the bundle: {relative}") + candidate = root + for part in relative.parts: + candidate = candidate / part + if candidate.is_symlink(): + raise PreflightError(f"trace output path must not use symlinks: {relative}") + try: + candidate.resolve().relative_to(root) + except ValueError as error: + raise PreflightError(f"trace output path is outside the bundle: {relative}") from error + return candidate + + +def read_proc_lines(path: Path) -> list[str]: + try: + return path.read_text(encoding="ascii").splitlines() + except OSError as error: + raise PreflightError(f"cannot read {path}: {error}") from error + + +def swap_audit() -> dict[str, object]: + lines = read_proc_lines(Path("/proc/swaps")) + entries = [] + for line in lines[1:]: + fields = line.split() + if len(fields) >= 5: + entries.append({ + "path": fields[0], + "type": fields[1], + "size_kib": int(fields[2]), + "used_kib": int(fields[3]), + "priority": int(fields[4]), + }) + return {"enabled": bool(entries), "entries": entries} + + +def memory_audit() -> dict[str, int]: + values: dict[str, int] = {} + for line in read_proc_lines(Path("/proc/meminfo")): + key, value = line.split(":", 1) + fields = value.split() + if fields: + values[key] = int(fields[0]) * 1024 + required = ("MemTotal", "MemAvailable") + if any(key not in values for key in required): + raise PreflightError("/proc/meminfo lacks MemTotal or MemAvailable") + result = { + "mem_total_bytes": values["MemTotal"], + "mem_available_bytes": values["MemAvailable"], + "mem_used_bytes": values["MemTotal"] - values["MemAvailable"], + } + if result["mem_used_bytes"] >= SOFT_MEMORY_LIMIT: + raise PreflightError( + f"host memory use is at or above the 116 GiB soft limit: {result['mem_used_bytes']}") + return result + + +def _command_text(*args: str) -> str: + try: + return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"{' '.join(args)} failed: {error}") from error + + +def darwin_host_and_memory_audit( + *, + command_text: Callable[..., str] = _command_text, + system: str | None = None, + machine: str | None = None) -> tuple[dict[str, object], dict[str, int]]: + platform_name = platform.system() if system is None else system + machine_name = platform.machine() if machine is None else machine + if platform_name != "Darwin" or machine_name != "arm64": + raise PreflightError("ds4 oracle execution requires macOS on arm64") + try: + total = int(command_text("sysctl", "-n", "hw.memsize")) + except ValueError as error: + raise PreflightError("Darwin memory size is invalid") from error + if total < 128 * 1024 * 1024 * 1024: + raise PreflightError("ds4 oracle host must have at least 128 GiB of memory") + vm_stat = command_text("vm_stat") + page_size_match = re.search(r"page size of ([0-9]+) bytes", vm_stat) + if page_size_match is None: + raise PreflightError("vm_stat page size is missing") + page_size = int(page_size_match.group(1)) + pages = {} + for name, value in re.findall(r"^([^:]+):\s+([0-9]+)\.$", vm_stat, flags=re.MULTILINE): + pages[name] = int(value) + available_pages = sum(pages.get(name, 0) for name in ( + "Pages free", + "Pages inactive", + "Pages speculative", + )) + available = available_pages * page_size + if available <= 0 or available > total: + raise PreflightError("Darwin available memory evidence is invalid") + host = { + "format": "dsv41-host-attestation", + "version": 1, + "runtime_kind": "apple-metal", + "platform": "macos", + "machine": "arm64", + "hardware_model": command_text("sysctl", "-n", "hw.model"), + "os_version": command_text("sysctl", "-n", "kern.osproductversion"), + "memory_bytes": total, + "source": "darwin-sysctl", + } + if not host["hardware_model"] or not host["os_version"]: + raise PreflightError("Darwin host identity is incomplete") + return host, { + "mem_total_bytes": total, + "mem_available_bytes": available, + "mem_used_bytes": total - available, + } + + +def darwin_swap_audit( + *, + command_text: Callable[..., str] = _command_text) -> dict[str, object]: + value = command_text("sysctl", "-n", "vm.swapusage") + match = re.fullmatch( + r"total = ([0-9]+(?:\.[0-9]+)?)M\s+used = ([0-9]+(?:\.[0-9]+)?)M\s+" + r"free = ([0-9]+(?:\.[0-9]+)?)M(?:\s+\(encrypted\))?", + value, + ) + if match is None: + raise PreflightError("Darwin swap evidence is invalid") + total, used, free = (int(float(item) * 1024 * 1024) for item in match.groups()) + if total != 0 or used != 0 or free != 0: + raise PreflightError("swap is enabled; model execution is blocked") + return { + "source": "darwin-sysctl-vm.swapusage", + "total_bytes": total, + "used_bytes": used, + "free_bytes": free, + } + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def proc_start_time_ticks(stat: str) -> int: + command_end = stat.rfind(")") + if command_end < 0: + raise PreflightError("watchdog process stat is invalid") + fields = stat[command_end + 2:].split() + if len(fields) < 20: + raise PreflightError("watchdog process stat is truncated") + return int(fields[19]) + + +def proc_parent_pid(stat: str) -> int: + command_end = stat.rfind(")") + if command_end < 0: + raise PreflightError("process stat is invalid") + fields = stat[command_end + 2:].split() + if len(fields) < 2: + raise PreflightError("process stat is truncated") + return int(fields[1]) + + +def proc_process_group_id(stat: str) -> int: + command_end = stat.rfind(")") + if command_end < 0: + raise PreflightError("process stat is invalid") + fields = stat[command_end + 2:].split() + if len(fields) < 3: + raise PreflightError("process stat is truncated") + return int(fields[2]) + + +def is_descendant(pid: int, ancestor_pid: int, procfs_root: Path) -> bool: + seen = set() + while pid > 1 and pid not in seen: + if pid == ancestor_pid: + return True + seen.add(pid) + try: + pid = proc_parent_pid((procfs_root / str(pid) / "stat").read_text(encoding="ascii")) + except (OSError, ValueError): + return False + return False + + +def open_watchdog_namespace_authority( + watchdog: dict[str, object], + *, + procfs_root: Path = Path("/proc"), + pidfd_open: Callable[[int, int], int] | None = None, +) -> tuple[int, dict[str, object]]: + if sys.platform != "linux": + raise PreflightError("watchdog namespace authority requires Linux pidfds") + opener = pidfd_open or getattr(os, "pidfd_open", None) + if opener is None: + raise PreflightError("watchdog namespace authority requires os.pidfd_open") + try: + watchdog_pid = int(watchdog["watchdog_pid"]) + watchdog_start = int(watchdog["watchdog_start_time_ticks"]) + guardian_pid = int(watchdog["guardian_pid"]) + child_pid = int(watchdog["child_pid"]) + child_pgid = int(watchdog["child_process_group_id"]) + executable_path = str(watchdog["watchdog_executable_path"]) + command_sha256 = str(watchdog["watchdog_command_sha256"]) + except (KeyError, TypeError, ValueError) as error: + raise PreflightError(f"watchdog namespace authority identity is invalid: {error}") from error + descriptor = -1 + try: + descriptor = int(opener(watchdog_pid, 0)) + watchdog_stat = (procfs_root / str(watchdog_pid) / "stat").read_text(encoding="ascii") + guardian_stat = (procfs_root / str(guardian_pid) / "stat").read_text(encoding="ascii") + child_stat = (procfs_root / str(child_pid) / "stat").read_text(encoding="ascii") + live_executable = str((procfs_root / str(watchdog_pid) / "exe").resolve(strict=True)) + live_command = (procfs_root / str(watchdog_pid) / "cmdline").read_bytes() + fdinfo = (procfs_root / "self" / "fdinfo" / str(descriptor)).read_text(encoding="ascii") + fdinfo_pid = next( + int(line.split(":", 1)[1].strip()) + for line in fdinfo.splitlines() + if line.startswith("Pid:") + ) + if proc_start_time_ticks(watchdog_stat) != watchdog_start: + raise PreflightError("watchdog namespace authority start time does not match") + if live_executable != str(Path(executable_path).resolve(strict=True)): + raise PreflightError("watchdog namespace authority executable does not match") + if sha256_bytes(live_command) != command_sha256: + raise PreflightError("watchdog namespace authority command does not match") + if fdinfo_pid != watchdog_pid: + raise PreflightError("watchdog namespace authority pidfd does not match") + watchdog_pgid = proc_process_group_id(watchdog_stat) + if proc_parent_pid(guardian_stat) != watchdog_pid or ( + proc_process_group_id(guardian_stat) != child_pgid) or ( + guardian_pid != child_pgid) or ( + proc_parent_pid(child_stat) != guardian_pid) or ( + proc_process_group_id(child_stat) != child_pgid): + raise PreflightError("watchdog namespace authority process tree does not match") + authority = { + "format": "dsv41-watchdog-namespace-authority", + "version": 1, + "mechanism": "inherited-pidfd", + "descriptor": descriptor, + "host_procfs_root": str(procfs_root.resolve(strict=True)), + "watchdog_pid": watchdog_pid, + "watchdog_process_group_id": watchdog_pgid, + "watchdog_start_time_ticks": watchdog_start, + "watchdog_executable_path": live_executable, + "watchdog_command_sha256": command_sha256, + "guardian_pid": guardian_pid, + "child_pid": child_pid, + "child_process_group_id": child_pgid, + } + return descriptor, authority + except (OSError, StopIteration, ValueError) as error: + if descriptor >= 0: + os.close(descriptor) + raise PreflightError(f"cannot establish watchdog namespace authority: {error}") from error + except BaseException: + if descriptor >= 0: + os.close(descriptor) + raise + + +def verify_watchdog_namespace_authority( + descriptor: int, + authority: dict[str, object], + *, + procfs_root: Path = Path("/proc"), +) -> None: + try: + if authority.get("format") != "dsv41-watchdog-namespace-authority" or ( + authority.get("version") != 1) or authority.get("mechanism") != "inherited-pidfd" or ( + authority.get("descriptor") != descriptor): + raise PreflightError("watchdog namespace authority receipt is invalid") + watchdog_pid = int(authority["watchdog_pid"]) + watchdog_start = int(authority["watchdog_start_time_ticks"]) + stat_text = (procfs_root / str(watchdog_pid) / "stat").read_text(encoding="ascii") + executable = str((procfs_root / str(watchdog_pid) / "exe").resolve(strict=True)) + command = (procfs_root / str(watchdog_pid) / "cmdline").read_bytes() + fdinfo = (procfs_root / "self" / "fdinfo" / str(descriptor)).read_text(encoding="ascii") + fdinfo_pid = next( + int(line.split(":", 1)[1].strip()) + for line in fdinfo.splitlines() + if line.startswith("Pid:") + ) + if fdinfo_pid != watchdog_pid or proc_start_time_ticks(stat_text) != watchdog_start or ( + executable != authority.get("watchdog_executable_path")) or ( + sha256_bytes(command) != authority.get("watchdog_command_sha256")): + raise PreflightError("watchdog namespace authority changed") + except PreflightError: + raise + except (KeyError, OSError, StopIteration, TypeError, ValueError) as error: + raise PreflightError(f"cannot verify watchdog namespace authority: {error}") from error + + +def read_heartbeat( + path: Path, + max_age_seconds: float, + *, + lease_id: str, + watchdog_pid: int, + watchdog_start_time_ticks: int, + child_pid: int, + child_pgid: int, + now: int | None = None, + monotonic_ns: Callable[[], int] = time.monotonic_ns) -> int: + try: + record = strict_json_loads(path.read_text(encoding="ascii")) + updated = datetime.fromisoformat(str(record["updated_at"]).replace("Z", "+00:00")) + heartbeat = int(updated.timestamp()) + except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + raise PreflightError(f"watchdog heartbeat is invalid: {error}") from error + if record.get("format") != WATCHDOG_HEARTBEAT_FORMAT or record.get("version") != WATCHDOG_VERSION: + raise PreflightError("watchdog heartbeat format is invalid") + if record.get("lease_id") != lease_id or record.get("state") != "active": + raise PreflightError("watchdog heartbeat lease identity or state is invalid") + if type(record.get("sequence")) is not int or record["sequence"] < 0: + raise PreflightError("watchdog heartbeat sequence is invalid") + if type(record.get("updated_monotonic_ns")) is not int or record["updated_monotonic_ns"] <= 0: + raise PreflightError("watchdog heartbeat monotonic timestamp is invalid") + if record.get("watchdog_pid") != watchdog_pid or ( + record.get("watchdog_start_time_ticks") != watchdog_start_time_ticks): + raise PreflightError("watchdog heartbeat owner does not match the lease") + if record.get("child_pid") != child_pid or record.get("child_process_group_id") != child_pgid: + raise PreflightError("watchdog heartbeat child identity does not match the lease") + age_ns = monotonic_ns() - record["updated_monotonic_ns"] + if age_ns < 0 or age_ns > int(max_age_seconds * 1_000_000_000): + raise PreflightError("watchdog heartbeat is stale") + now = int(time.time()) if now is None else now + if heartbeat <= 0 or heartbeat > now: + raise PreflightError("watchdog heartbeat wall-clock timestamp is invalid") + return heartbeat + + +def _read_json_with_retry( + path: Path, + *, + timeout_seconds: float, + monotonic: Callable[[], float], + sleeper: Callable[[float], None]) -> dict[str, object]: + deadline = monotonic() + timeout_seconds + last_error: Exception | None = None + while True: + try: + record = strict_json_loads(path.read_text(encoding="ascii")) + if not isinstance(record, dict): + raise ValueError("record is not an object") + return record + except (OSError, ValueError, TypeError, json.JSONDecodeError) as error: + last_error = error + if monotonic() >= deadline: + raise PreflightError(f"watchdog lease did not become ready: {last_error}") from last_error + sleeper(0.05) + + +def _read_watchdog_events(path: Path) -> list[dict[str, object]]: + try: + lines = path.read_text(encoding="ascii").splitlines() + except OSError as error: + raise PreflightError(f"cannot read watchdog audit: {error}") from error + events = [] + for line_number, line in enumerate(lines, start=1): + try: + event = validate_watchdog_event(strict_json_loads(line)) + except (PreflightError, TraceError) as error: + raise PreflightError(f"watchdog audit line {line_number} is invalid: {error}") from error + events.append(event) + if not events: + raise PreflightError("watchdog audit is empty") + return events + + +def _read_watchdog_startup_events( + path: Path, + *, + timeout_seconds: float, + monotonic: Callable[[], float], + sleeper: Callable[[float], None]) -> list[dict[str, object]]: + deadline = monotonic() + timeout_seconds + last_error: Exception | None = None + while True: + try: + events = _read_watchdog_events(path) + if any(event.get("event") == "preflight" for event in events) and ( + any(event.get("event") == "child_started" for event in events)): + return events + last_error = PreflightError("watchdog audit lacks preflight or child_started evidence") + except PreflightError as error: + last_error = error + if monotonic() >= deadline: + assert last_error is not None + raise PreflightError(f"watchdog audit did not become ready: {last_error}") from last_error + sleeper(0.05) + + +def _load_watchdog_module(script: Path) -> object: + spec = importlib.util.spec_from_file_location("dsv41_strix_memory_watchdog", script) + if spec is None or spec.loader is None: + raise PreflightError(f"cannot load canonical watchdog module: {script}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError, RuntimeError, SyntaxError) as error: + raise PreflightError(f"cannot load canonical watchdog module: {error}") from error + return module + + +def _watchdog_audit_result( + lease: dict[str, object], + *, + watchdog_revision: str, + lease_path: Path, + heartbeat_path: Path, + audit_path: Path, + audit_event_count: int, + procfs_root: Path = Path("/proc")) -> dict[str, object]: + try: + heartbeat_record = strict_json_loads(heartbeat_path.read_text(encoding="ascii")) + updated = datetime.fromisoformat(str(heartbeat_record["updated_at"]).replace("Z", "+00:00")) + heartbeat_unix = int(updated.timestamp()) + except (OSError, UnicodeError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + raise PreflightError(f"canonical watchdog heartbeat is invalid: {error}") from error + try: + audit_sha256 = sha256_bytes(audit_path.read_bytes()) + except OSError as error: + raise PreflightError(f"cannot hash canonical watchdog audit: {error}") from error + watchdog_command = lease.get("watchdog_command") + if not isinstance(watchdog_command, str) or not watchdog_command: + try: + command_bytes = ( + procfs_root / str(lease["watchdog_pid"]) / "cmdline").read_bytes() + except (OSError, KeyError) as error: + raise PreflightError(f"cannot read canonical watchdog command: {error}") from error + watchdog_command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() + if not watchdog_command: + raise PreflightError("canonical watchdog command is empty") + result = dict(lease) + result.update({ + "watchdog_revision": watchdog_revision, + "lease_path": str(lease_path), + "watchdog_command": watchdog_command, + "heartbeat_unix": heartbeat_unix, + "audit_live_path": str(audit_path), + "audit_path": str(audit_path), + "audit_sha256": audit_sha256, + "audit_event_count": audit_event_count, + }) + return result + + +def _canonical_watchdog_audit( + repo: Path, + *, + environment: dict[str, str] | os._Environ[str], + current_pid: int | None, + timeout_seconds: float, + monotonic: Callable[[], float], + sleeper: Callable[[float], None], + watchdog_module: object | None) -> dict[str, object]: + global _WATCHDOG_GUARD + try: + lease_path = require_nvme_path(Path(environment[WATCHDOG_LEASE_ENV]), "watchdog lease") + heartbeat_path = require_nvme_path(Path(environment[WATCHDOG_HEARTBEAT_ENV]), "watchdog heartbeat") + audit_path = require_nvme_path(Path(environment[WATCHDOG_AUDIT_ENV]), "watchdog audit") + max_age_seconds = float(environment[WATCHDOG_MAX_AGE_ENV]) + except (KeyError, ValueError) as error: + raise PreflightError(f"canonical watchdog environment is incomplete: {error}") from error + if max_age_seconds <= 0 or max_age_seconds > MAX_WATCHDOG_HEARTBEAT_AGE: + raise PreflightError(f"watchdog heartbeat age must be within 1..{MAX_WATCHDOG_HEARTBEAT_AGE} seconds") + expected_script = resolved(repo / "scripts" / "strix_memory_watchdog.py") + if not expected_script.is_file(): + raise PreflightError(f"canonical watchdog script is missing: {expected_script}") + script_sha256 = sha256_bytes(expected_script.read_bytes()) + watchdog_revision = APPROVED_WATCHDOGS.get(script_sha256) + if watchdog_revision is None: + raise PreflightError( + "no independently reviewed watchdog revision is approved for correctness execution") + module = watchdog_module or _load_watchdog_module(expected_script) + validator = getattr(module, "validate_active_lease", None) + validation_error = getattr(module, "LeaseValidationError", RuntimeError) + guard = getattr(module, "start_process_group_lease_guard", None) + if not callable(validator) or not isinstance(validation_error, type): + raise PreflightError("canonical watchdog module lacks the lease validation API") + process_id = os.getpid() if current_pid is None else current_pid + try: + lease = validator( + lease_path, + expected_script_path=expected_script, + expected_soft_bytes=SOFT_MEMORY_LIMIT, + expected_emergency_bytes=WATCHDOG_EMERGENCY_LIMIT, + expected_procfs_root=Path("/proc"), + expected_heartbeat_path=heartbeat_path, + expected_audit_path=audit_path, + expected_max_heartbeat_age_seconds=max_age_seconds, + current_process_id=process_id, + process_procfs_root=Path("/proc"), + ) + if lease.get("child_pid") == process_id and _WATCHDOG_GUARD is None: + if not callable(guard): + raise PreflightError("canonical watchdog module lacks the process-group lease guard") + _WATCHDOG_GUARD = guard( + expected_script, + startup_timeout_seconds=timeout_seconds, + expected_procfs_root=Path("/proc"), + process_procfs_root=Path("/proc"), + ) + except validation_error as error: + raise PreflightError(f"canonical watchdog lease is invalid: {error}") from error + if not isinstance(lease, dict): + raise PreflightError("canonical watchdog lease validator returned invalid data") + if lease.get("grace_seconds") != 30.0 or lease.get("sample_interval_seconds") != 1.0: + raise PreflightError("canonical watchdog timing policy is invalid") + events = _read_watchdog_startup_events( + audit_path, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + ) + return _watchdog_audit_result( + lease, + watchdog_revision=watchdog_revision, + lease_path=lease_path, + heartbeat_path=heartbeat_path, + audit_path=audit_path, + audit_event_count=len(events), + ) + + +def watchdog_audit( + repo: Path, + *, + environment: dict[str, str] | os._Environ[str] = os.environ, + procfs_root: Path = Path("/proc"), + current_pid: int | None = None, + current_pgid: int | None = None, + getpgid: Callable[[int], int] = os.getpgid, + now: int | None = None, + monotonic_ns: Callable[[], int] = time.monotonic_ns, + timeout_seconds: float = WATCHDOG_STARTUP_TIMEOUT_SECONDS, + monotonic: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, + watchdog_module: object | None = None) -> dict[str, object]: + if watchdog_module is not None or ( + procfs_root == Path("/proc") and current_pid is None and current_pgid is None): + return _canonical_watchdog_audit( + repo, + environment=environment, + current_pid=current_pid, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + watchdog_module=watchdog_module, + ) + try: + pid_file = Path(environment[WATCHDOG_LEASE_ENV]) + environment_heartbeat = require_nvme_path( + Path(environment[WATCHDOG_HEARTBEAT_ENV]), "watchdog heartbeat") + environment_audit = require_nvme_path( + Path(environment[WATCHDOG_AUDIT_ENV]), "watchdog audit") + environment_max_age = float(environment[WATCHDOG_MAX_AGE_ENV]) + except (KeyError, ValueError) as error: + raise PreflightError(f"canonical watchdog environment is incomplete: {error}") from error + pid_file = require_nvme_path(pid_file, "watchdog lease") + repo = resolved(repo) + expected_script = resolved(repo / "scripts" / "strix_memory_watchdog.py") + if not expected_script.is_file(): + raise PreflightError(f"canonical watchdog script is missing: {expected_script}") + expected_script_sha256 = sha256_bytes(expected_script.read_bytes()) + lease = _read_json_with_retry( + pid_file, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + ) + try: + pid = int(lease["watchdog_pid"]) + expected_start = int(lease["watchdog_start_time_ticks"]) + expected_command_sha256 = str(lease["watchdog_command_sha256"]) + child_pid = int(lease["child_pid"]) + child_pgid = int(lease["child_process_group_id"]) + script_path = resolved(Path(str(lease["watchdog_script_path"]))) + script_sha256 = str(lease["watchdog_script_sha256"]) + lease_id = str(lease["lease_id"]) + soft_bytes = int(lease["soft_bytes"]) + emergency_bytes = int(lease["emergency_bytes"]) + strict_bytes = int(lease["strict_ceiling_bytes"]) + procfs_path = str(lease["procfs_root"]) + heartbeat_path = require_nvme_path(Path(lease["heartbeat_path"]), "watchdog heartbeat") + audit_path = require_nvme_path(Path(lease["audit_path"]), "watchdog audit") + max_age_seconds = float(lease.get("max_heartbeat_age_seconds", MAX_WATCHDOG_HEARTBEAT_AGE)) + except (OSError, ValueError, TypeError, KeyError) as error: + raise PreflightError(f"watchdog lease is invalid: {error}") from error + if lease.get("format") != WATCHDOG_LEASE_FORMAT or lease.get("version") != WATCHDOG_VERSION: + raise PreflightError("watchdog lease format is invalid") + if lease.get("state") != "active" or re.fullmatch(r"[0-9a-f]{32,64}", lease_id) is None: + raise PreflightError("watchdog lease identity or state is invalid") + if script_path != expected_script or script_sha256 != expected_script_sha256: + raise PreflightError("watchdog script identity does not match the candidate repository") + if soft_bytes != SOFT_MEMORY_LIMIT or emergency_bytes != WATCHDOG_EMERGENCY_LIMIT or ( + strict_bytes != STRICT_MEMORY_LIMIT): + raise PreflightError("watchdog memory thresholds are invalid") + if procfs_path != "/proc": + raise PreflightError("watchdog procfs root must be /proc") + if heartbeat_path != environment_heartbeat or audit_path != environment_audit or ( + max_age_seconds != environment_max_age): + raise PreflightError("watchdog lease paths or heartbeat age do not match the inherited environment") + if re.fullmatch(r"[0-9a-f]{64}", expected_command_sha256) is None: + raise PreflightError("watchdog lease command SHA-256 is invalid") + if max_age_seconds <= 0 or max_age_seconds > MAX_WATCHDOG_HEARTBEAT_AGE: + raise PreflightError(f"watchdog heartbeat age must be within 1..{MAX_WATCHDOG_HEARTBEAT_AGE} seconds") + if pid <= 1 or child_pid <= 1 or child_pgid <= 1: + raise PreflightError("watchdog or monitored child identity is invalid") + if current_pgid is None: + current_pgid = os.getpgrp() + if current_pid is None: + current_pid = os.getpid() + if current_pgid != child_pgid: + raise PreflightError("current process is outside the watchdog-monitored process group") + if not is_descendant(current_pid, child_pid, procfs_root): + raise PreflightError("current process is not a descendant of the watchdog-monitored child") + try: + child_parent = proc_parent_pid( + (procfs_root / str(child_pid) / "stat").read_text(encoding="ascii")) + if child_parent != pid or getpgid(child_pid) != child_pgid or child_pgid != child_pid: + raise PreflightError("watchdog child process group does not match the lease") + except (OSError, ValueError) as error: + raise PreflightError(f"cannot inspect watchdog child process group: {error}") from error + if not (procfs_root / str(pid)).exists(): + raise PreflightError(f"watchdog process {pid} is not running") + try: + command_bytes = (procfs_root / str(pid) / "cmdline").read_bytes() + start_time_ticks = proc_start_time_ticks( + (procfs_root / str(pid) / "stat").read_text(encoding="ascii")) + except (OSError, ValueError) as error: + raise PreflightError(f"cannot inspect watchdog process {pid}: {error}") from error + command_sha256 = sha256_bytes(command_bytes) + if start_time_ticks != expected_start or command_sha256 != expected_command_sha256: + raise PreflightError("watchdog process identity does not match its lease") + command_parts = [part.decode("utf-8", "replace") for part in command_bytes.split(b"\0") if part] + try: + watchdog_cwd = (procfs_root / str(pid) / "cwd").resolve() + except OSError as error: + raise PreflightError(f"cannot inspect watchdog process working directory: {error}") from error + script_named = any( + resolved(Path(argument) if Path(argument).is_absolute() else watchdog_cwd / argument) == expected_script + for argument in command_parts + ) + if not script_named: + raise PreflightError("watchdog command does not execute the candidate repository script") + child_command = lease.get("command") + if not isinstance(child_command, list) or not child_command or ( + not all(isinstance(argument, str) for argument in child_command)): + raise PreflightError("watchdog child command is invalid") + child_command_sha256 = sha256_bytes( + json.dumps(child_command, ensure_ascii=True, separators=(",", ":")).encode("utf-8")) + if child_command_sha256 != lease.get("child_command_sha256"): + raise PreflightError("watchdog child command does not match the lease") + heartbeat = read_heartbeat( + heartbeat_path, + max_age_seconds, + lease_id=lease_id, + watchdog_pid=pid, + watchdog_start_time_ticks=start_time_ticks, + child_pid=child_pid, + child_pgid=child_pgid, + now=now, + monotonic_ns=monotonic_ns, + ) + command = command_bytes.replace(b"\0", b" ").decode("utf-8", "replace").strip() + if not command: + raise PreflightError(f"watchdog process {pid} has no command line") + events = _read_watchdog_startup_events( + audit_path, + timeout_seconds=timeout_seconds, + monotonic=monotonic, + sleeper=sleeper, + ) + preflight = next((event for event in events if event.get("event") == "preflight"), None) + child_started = next((event for event in events if event.get("event") == "child_started"), None) + if preflight is None or child_started is None: + raise PreflightError("watchdog audit lacks preflight or child_started evidence") + if preflight.get("soft_bytes") != SOFT_MEMORY_LIMIT or ( + preflight.get("emergency_bytes") != WATCHDOG_EMERGENCY_LIMIT) or ( + preflight.get("strict_ceiling_bytes") != STRICT_MEMORY_LIMIT): + raise PreflightError("watchdog audit thresholds are invalid") + if preflight.get("swap_entries") != 0: + raise PreflightError("watchdog audit does not report zero swap") + if child_started.get("child_pid") != child_pid or ( + child_started.get("process_group_id") != child_pgid): + raise PreflightError("watchdog audit child identity does not match the lease") + if child_started.get("command") != lease.get("command"): + raise PreflightError("watchdog audit child command does not match the lease") + return { + "format": WATCHDOG_LEASE_FORMAT, + "version": WATCHDOG_VERSION, + "lease_id": lease_id, + "lease_path": str(pid_file), + "watchdog_pid": pid, + "watchdog_start_time_ticks": start_time_ticks, + "watchdog_command": command, + "watchdog_command_sha256": command_sha256, + "watchdog_script_path": str(script_path), + "watchdog_script_sha256": script_sha256, + "soft_bytes": soft_bytes, + "emergency_bytes": emergency_bytes, + "strict_ceiling_bytes": strict_bytes, + "procfs_root": procfs_path, + "child_pid": child_pid, + "child_process_group_id": child_pgid, + "command": child_command, + "child_command_sha256": child_command_sha256, + "heartbeat_path": str(heartbeat_path), + "heartbeat_unix": heartbeat, + "max_heartbeat_age_seconds": max_age_seconds, + "audit_path": str(audit_path), + "audit_sha256": sha256_bytes(audit_path.read_bytes()), + "audit_event_count": len(events), + } + + +def process_ancestry(pid: int, procfs_root: Path) -> set[int]: + ancestors = {pid} + while pid > 1: + try: + parent = proc_parent_pid((procfs_root / str(pid) / "stat").read_text(encoding="ascii")) + except (OSError, UnicodeError, ValueError): + break + if parent <= 1 or parent in ancestors: + break + ancestors.add(parent) + pid = parent + return ancestors + + +def matching_workloads( + patterns: list[str], + *, + procfs_root: Path = Path("/proc"), + current_pid: int | None = None) -> list[dict[str, object]]: + matches = [] + excluded = process_ancestry(os.getpid() if current_pid is None else current_pid, procfs_root) + lowered = [pattern.lower() for pattern in patterns if pattern] + for entry in procfs_root.iterdir(): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + if pid in excluded: + continue + try: + command = (entry / "cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace").strip() + except OSError: + continue + command_lower = command.lower() + if any(pattern in command_lower for pattern in lowered): + matches.append({"pid": pid, "command": command}) + return matches + + +def darwin_matching_workloads( + patterns: list[str], + *, + command_text: Callable[..., str] = _command_text, + current_pid: int | None = None) -> list[dict[str, object]]: + pid_to_parent = {} + pid_to_command = {} + for line in command_text("ps", "-axo", "pid=,ppid=,command=").splitlines(): + fields = line.strip().split(maxsplit=2) + if len(fields) != 3: + continue + try: + pid = int(fields[0]) + parent = int(fields[1]) + except ValueError: + continue + pid_to_parent[pid] = parent + pid_to_command[pid] = fields[2] + excluded = set() + pid = os.getpid() if current_pid is None else current_pid + while pid > 0 and pid not in excluded: + excluded.add(pid) + pid = pid_to_parent.get(pid, 0) + lowered = [pattern.lower() for pattern in patterns if pattern] + return [ + {"pid": pid, "command": command} + for pid, command in pid_to_command.items() + if pid not in excluded and any(pattern in command.lower() for pattern in lowered) + ] + + +def run_strix_preflight( + *, + model: Path, + prompt: Path, + output: Path, + repo: Path, + busy_patterns: list[str], +) -> dict[str, object]: + model_storage = storage_attestation(model, "model") + prompt_storage = storage_attestation(prompt, "prompt") + output_storage = storage_attestation(output, "trace output") + repo_storage = storage_attestation(repo, "repository") + tmpdir_value = os.environ.get("TMPDIR") + if not tmpdir_value: + raise PreflightError("TMPDIR is required for NVMe-only correctness runs") + tmpdir_input = require_safe_tmpdir_path(Path(tmpdir_value)) + tmp_storage = storage_attestation(tmpdir_input, "temporary directory") + tmpdir = _attested_resolved_path(tmp_storage, "temporary directory") + if not tmpdir.is_dir() or not os.access(tmpdir, os.W_OK | os.X_OK): + raise PreflightError("TMPDIR must be an existing writable directory") + model = _attested_resolved_path(model_storage, "model") + prompt = _attested_resolved_path(prompt_storage, "prompt") + output = _attested_resolved_path(output_storage, "trace output") + if not model.is_file(): + raise PreflightError(f"model is not a file: {model}") + if not prompt.is_file(): + raise PreflightError(f"prompt is not a file: {prompt}") + if os.environ.get("HIP_LAUNCH_BLOCKING") != "1": + raise PreflightError("HIP_LAUNCH_BLOCKING=1 is required for gfx1151 correctness runs") + swap = swap_audit() + if swap["enabled"]: + raise PreflightError("swap is enabled; model execution is blocked") + watchdog = watchdog_audit(repo) + workloads = matching_workloads(busy_patterns) + if workloads: + raise PreflightError("active model workload detected: " + json.dumps(workloads, ensure_ascii=True)) + return { + "created_unix": int(time.time()), + "runtime_kind": "strix-rocm", + "model": str(model), + "prompt": str(prompt), + "output": str(output), + "memory": memory_audit(), + "swap": swap, + "watchdog": watchdog, + "active_workloads": [], + "environment": {"HIP_LAUNCH_BLOCKING": "1"}, + "storage_policy": dict(NO_EXTERNAL_STATE_STORAGE), + "storage": { + "model": model_storage, + "prompt": prompt_storage, + "output": output_storage, + "repository": repo_storage, + "temporary_directory": tmp_storage, + }, + } + + +def run_oracle_preflight( + *, + model: Path, + prompt: Path, + output: Path, + repo: Path, + checkout: Path, + busy_patterns: list[str], + accelerator: dict[str, object], + runner: dict[str, object], + disk_info: Callable[[Path], dict[str, object]] = _diskutil_info, + command_text: Callable[..., str] = _command_text, + system: str | None = None, + machine: str | None = None) -> dict[str, object]: + model_storage = darwin_storage_attestation(model, "model", disk_info=disk_info) + prompt_storage = darwin_storage_attestation(prompt, "prompt", disk_info=disk_info) + output_storage = darwin_storage_attestation(output, "trace output", disk_info=disk_info) + repo_storage = darwin_storage_attestation(repo, "repository", disk_info=disk_info) + checkout_storage = darwin_storage_attestation(checkout, "ds4 checkout", disk_info=disk_info) + runner_executable = Path(str(runner.get("runner_executable", ""))) + runner_script = Path(str(runner.get("runner_script", ""))) + exporter = Path(str(runner.get("exporter_path", ""))) + runner_executable_storage = darwin_storage_attestation( + runner_executable, "runner executable", disk_info=disk_info) + runner_script_storage = darwin_storage_attestation(runner_script, "runner script", disk_info=disk_info) + exporter_storage = darwin_storage_attestation(exporter, "trace exporter", disk_info=disk_info) + tmpdir_value = os.environ.get("TMPDIR") + if not tmpdir_value: + raise PreflightError("TMPDIR is required for ds4 oracle correctness runs") + tmpdir_input = require_safe_tmpdir_path(Path(tmpdir_value)) + tmp_storage = darwin_storage_attestation(tmpdir_input, "temporary directory", disk_info=disk_info) + tmpdir = _attested_resolved_path(tmp_storage, "temporary directory") + if not tmpdir.is_dir() or not os.access(tmpdir, os.W_OK | os.X_OK): + raise PreflightError("TMPDIR must be an existing writable directory") + model = _attested_resolved_path(model_storage, "model") + prompt = _attested_resolved_path(prompt_storage, "prompt") + output = _attested_resolved_path(output_storage, "trace output") + repo = _attested_resolved_path(repo_storage, "repository") + checkout = _attested_resolved_path(checkout_storage, "ds4 checkout") + runner_executable = _attested_resolved_path(runner_executable_storage, "runner executable") + runner_script = _attested_resolved_path(runner_script_storage, "runner script") + exporter = _attested_resolved_path(exporter_storage, "trace exporter") + if not model.is_file(): + raise PreflightError(f"model is not a file: {model}") + if not prompt.is_file(): + raise PreflightError(f"prompt is not a file: {prompt}") + if not runner_executable.is_file() or not runner_script.is_file() or not exporter.is_file(): + raise PreflightError("ds4 runner or exporter path is not a file") + try: + runner_script.relative_to(repo) + except ValueError as error: + raise PreflightError("ds4 runner script is outside the attested repository") from error + for key, path in ( + ("runner_executable_sha256", runner_executable), + ("runner_script_sha256", runner_script), + ("exporter_sha256", exporter)): + try: + digest = sha256_bytes(path.read_bytes()) + except OSError as error: + raise PreflightError(f"cannot hash ds4 {key} path: {error}") from error + if runner.get(key) != digest: + raise PreflightError(f"ds4 {key} differs from the executed file") + if runner.get("checkout_path") != str(checkout): + raise PreflightError("ds4 runner checkout path differs from the attested checkout") + host, memory = darwin_host_and_memory_audit( + command_text=command_text, + system=system, + machine=machine, + ) + swap = darwin_swap_audit(command_text=command_text) + workloads = darwin_matching_workloads(busy_patterns, command_text=command_text) + if workloads: + raise PreflightError("active model workload detected: " + json.dumps(workloads, ensure_ascii=True)) + return { + "created_unix": int(time.time()), + "runtime_kind": "apple-metal", + "model": str(model), + "prompt": str(prompt), + "output": str(output), + "memory": memory, + "swap": swap, + "runner": runner, + "host": host, + "accelerator": accelerator, + "active_workloads": [], + "environment": {}, + "storage_policy": dict(NO_EXTERNAL_STATE_STORAGE), + "storage": { + "model": model_storage, + "prompt": prompt_storage, + "output": output_storage, + "repository": repo_storage, + "runtime_checkout": checkout_storage, + "temporary_directory": tmp_storage, + "runner_executable": runner_executable_storage, + "runner_script": runner_script_storage, + "exporter": exporter_storage, + }, + } + + +def write_audits(root: Path, audit: dict[str, object]) -> dict[str, str]: + root = resolved(root) + if root.exists() and any(root.iterdir()): + raise PreflightError(f"audit directory is not empty: {root}") + root.mkdir(parents=True, exist_ok=True) + result = {} + runtime_kind = audit.get("runtime_kind") + kinds = ( + ("memory", "swap", "watchdog") + if runtime_kind == "strix-rocm" + else ("memory", "swap", "runner") + if runtime_kind == "apple-metal" + else () + ) + if not kinds: + raise PreflightError("audit runtime kind is invalid") + for key in kinds: + path = root / f"{key}.json" + data = audit[key] + if key == "watchdog": + data = dict(data) + live_audit_path = resolved(Path(str(data["audit_path"]))) + try: + audit_bytes = live_audit_path.read_bytes() + audit_text = audit_bytes.decode("ascii") + except (OSError, UnicodeError) as error: + raise PreflightError(f"cannot snapshot watchdog audit: {error}") from error + events = [] + for line_number, line in enumerate(audit_text.splitlines(), start=1): + try: + event = validate_watchdog_event(strict_json_loads(line)) + except (PreflightError, TraceError) as error: + raise PreflightError( + f"watchdog audit line {line_number} is invalid while snapshotting: {error}") from error + events.append(event) + if not events: + raise PreflightError("watchdog audit snapshot is empty") + snapshot = root / "watchdog-events.jsonl" + snapshot.write_bytes(audit_bytes) + data["audit_live_path"] = str(live_audit_path) + data["audit_path"] = str(snapshot) + data["audit_sha256"] = sha256_bytes(audit_bytes) + data["audit_event_count"] = len(events) + value = { + "created_unix": audit["created_unix"], + "kind": key, + "data": data, + "environment": audit["environment"], + } + if key == "memory": + value["storage"] = audit["storage"] + value["storage_policy"] = audit["storage_policy"] + value["accelerator"] = audit["accelerator"] + if "host" in audit: + value["host"] = audit["host"] + path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + result[key] = str(path) + summary = root / "preflight.json" + summary.write_text(json.dumps(audit, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + result["preflight"] = str(summary) + return result + + +def seal_audits(audits: dict[str, str]) -> dict[str, str]: + digests = {} + paths = [] + try: + kinds = tuple(kind for kind in ("memory", "swap", "watchdog", "runner") if kind in audits) + if set(kinds) != set(audits) - {"preflight"}: + raise PreflightError("audit set has unsupported kinds") + for kind in kinds: + path = resolved(Path(audits[kind])) + paths.append(path) + data = path.read_bytes() + digests[kind] = sha256_bytes(data) + if kind == "watchdog": + record = strict_json_loads(data.decode("ascii")) + jsonl_path = resolved(Path(record["data"]["audit_path"])) + paths.append(jsonl_path) + digests["watchdog_jsonl"] = sha256_bytes(jsonl_path.read_bytes()) + for path in paths: + path.chmod(0o444) + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError) as error: + raise PreflightError(f"cannot seal audit evidence: {error}") from error + return digests + + +def verify_sealed_audits(audits: dict[str, str], digests: dict[str, str]) -> None: + current = seal_audits(audits) + if current != digests: + raise PreflightError("preflight audit evidence changed during runtime execution") + + +def embed_audits(trace_root: Path, phase: str, audits: dict[str, str]) -> dict[str, dict[str, object]]: + trace_root = safe_trace_path(trace_root, ".") + embedded_root = safe_trace_path(trace_root, Path("audits") / phase) + embedded_root.mkdir(parents=True, exist_ok=True) + result = {} + kinds = tuple(kind for kind in ("memory", "swap", "watchdog", "runner") if kind in audits) + if set(kinds) != set(audits) - {"preflight"}: + raise PreflightError("audit set has unsupported kinds") + for kind in kinds: + source = resolved(Path(audits[kind])) + data = source.read_bytes() + try: + record = strict_json_loads(data.decode("ascii")) + except (UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot embed {kind} audit: {error}") from error + if kind == "watchdog": + try: + jsonl_source = resolved(Path(record["data"].pop("audit_path"))) + jsonl_data = jsonl_source.read_bytes() + except (OSError, TypeError, KeyError) as error: + raise PreflightError(f"cannot embed watchdog JSONL audit: {error}") from error + jsonl_digest = sha256_bytes(jsonl_data) + if record["data"].get("audit_sha256") != jsonl_digest: + raise PreflightError("watchdog JSONL audit SHA-256 changed before embedding") + jsonl_destination = safe_trace_path( + trace_root, Path("audits") / phase / f"{jsonl_digest}.jsonl") + if jsonl_destination.exists() and jsonl_destination.read_bytes() != jsonl_data: + raise PreflightError(f"content-addressed watchdog audit collision: {jsonl_destination}") + if not jsonl_destination.exists(): + jsonl_destination.write_bytes(jsonl_data) + record["data"]["audit"] = { + "path": f"audits/{phase}/{jsonl_digest}.jsonl", + "sha256": jsonl_digest, + "event_count": record["data"].pop("audit_event_count"), + } + data = (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + digest = sha256_bytes(data) + destination = safe_trace_path(trace_root, Path("audits") / phase / f"{digest}.json") + if destination.exists() and destination.read_bytes() != data: + raise PreflightError(f"content-addressed audit collision: {destination}") + if not destination.exists(): + destination.write_bytes(data) + try: + created = int(record["created_unix"]) + except (ValueError, TypeError, KeyError) as error: + raise PreflightError(f"cannot embed {kind} audit: {error}") from error + result[kind] = { + "path": f"audits/{phase}/{digest}.json", + "sha256": digest, + "created_unix": created, + } + return result + + +def bind_embedded_audits(trace_root: Path, audit_sets: dict[str, dict[str, str]]) -> None: + trace_root = safe_trace_path(trace_root, ".") + manifest_path = safe_trace_path(trace_root, "manifest.json") + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot bind trace audits: {error}") from error + if set(audit_sets) != {"pre", "post"}: + raise PreflightError("trace requires pre and post audit sets") + manifest["audits"] = { + phase: embed_audits(trace_root, phase, audits) + for phase, audits in audit_sets.items() + } + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +def validate_prompt_provenance( + path: Path, + *, + prompt: Path, + corpus_name: str, + corpus_sha256: str, + model_sha256: str, + target_tokens: int, + context: int, + decode_steps: int, + builder_approval_id: str, + builder_policy: dict[str, object], + builder_policy_sha256: str, + path_resolver: Callable[[Path, str], Path] | None = None, +) -> dict[str, object]: + path = (require_nvme_path if path_resolver is None else path_resolver)(path, "prompt provenance") + try: + data = path.read_bytes() + record = strict_json_loads(data.decode("ascii")) + prompt_path = resolved(prompt) + prompt_bytes = prompt_path.read_bytes() + prompt_size = prompt_path.stat().st_size + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"prompt provenance is invalid: {error}") from error + if not isinstance(record, dict): + raise PreflightError("prompt provenance must be a JSON object") + source_root_lexical = Path(str(builder_policy["source_root"])) + source_root_resolved = resolved(source_root_lexical) + corpus_resolved = resolved(source_root_resolved / "tests" / "corpus" / corpus_name) + expected = { + "format": "dsv41-prompt-provenance", + "version": 2, + "corpus_name": corpus_name, + "corpus_sha256": corpus_sha256, + "corpus_path": str(corpus_resolved), + "corpus_resolved_path": str(corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), + "model_sha256": model_sha256, + "prompt_sha256": sha256_bytes(prompt_bytes), + "prompt_byte_count": prompt_size, + "context": context, + "decode_steps": decode_steps, + "target_tokens": target_tokens, + "actual_tokens": target_tokens, + "builder_approval_id": builder_approval_id, + "builder_approval_sha256": builder_policy_sha256, + "builder_path": builder_policy["executable_path"], + "builder_sha256": builder_policy["executable_sha256"], + "builder_revision": builder_policy["revision"], + "builder_runtime_profile": builder_policy["runtime_profile"], + "tokenizer": builder_policy["tokenizer"], + } + for key, value in expected.items(): + if record.get(key) != value: + raise PreflightError(f"prompt provenance {key} mismatch") + required = set(expected) | { + "corpus_lexical_path", + "builder_runtime_build", + "builder_runtime_build_sha256", + "builder_install_trust", + "builder_install_trust_sha256", + } + if set(record) != required: + raise PreflightError("prompt provenance fields are invalid") + try: + if resolved(Path(str(record["corpus_lexical_path"]))) != corpus_resolved: + raise PreflightError("prompt provenance corpus lexical path resolves outside the approved source") + except OSError as error: + raise PreflightError(f"prompt provenance corpus lexical path is invalid: {error}") from error + try: + runtime_build = validate_runtime_build_evidence( + record["builder_runtime_build"], builder_policy, label="prompt builder") + runtime_build_sha256 = runtime_build_evidence_sha256( + runtime_build, builder_policy, label="prompt builder") + trust = validate_install_trust_evidence(record["builder_install_trust"], builder_policy) + trust_sha256 = install_trust_sha256(trust) + except TraceError as error: + raise PreflightError(f"prompt provenance runtime trust is invalid: {error}") from error + if record["builder_runtime_build_sha256"] != runtime_build_sha256: + raise PreflightError("prompt provenance runtime build SHA-256 mismatch") + if record["builder_install_trust_sha256"] != trust_sha256: + raise PreflightError("prompt provenance install trust SHA-256 mismatch") + matches = [ + prompt_record for prompt_record in builder_policy["prompts"] + if prompt_record["corpus_name"] == corpus_name and + prompt_record["context"] == context and + prompt_record["decode_steps"] == decode_steps + ] + if len(matches) != 1: + raise PreflightError("prompt provenance configuration is not externally approved") + for key in ( + "corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", + "prompt_sha256", "prompt_byte_count"): + if record[key] != matches[0][key]: + raise PreflightError(f"prompt provenance {key} differs from external approval") + return {"path": str(path), "bytes": data, "record": record} + + +def bind_prompt_provenance(trace_root: Path, provenance: dict[str, object]) -> None: + trace_root = safe_trace_path(trace_root, ".") + manifest_path = safe_trace_path(trace_root, "manifest.json") + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise PreflightError(f"cannot bind prompt provenance: {error}") from error + prompt = manifest.get("prompt") + if not isinstance(prompt, dict): + raise PreflightError("trace manifest prompt is invalid") + record = provenance["record"] + data = provenance["bytes"] + if not isinstance(record, dict) or not isinstance(data, bytes): + raise PreflightError("validated prompt provenance is invalid") + digest = sha256_bytes(data) + provenance_root = safe_trace_path(trace_root, "provenance") + provenance_root.mkdir(parents=True, exist_ok=True) + destination = safe_trace_path(trace_root, Path("provenance") / f"{digest}.json") + if destination.exists() and destination.read_bytes() != data: + raise PreflightError(f"content-addressed provenance collision: {destination}") + if not destination.exists(): + destination.write_bytes(data) + prompt["corpus_name"] = record["corpus_name"] + prompt["corpus_sha256"] = record["corpus_sha256"] + prompt["target_tokens"] = record["target_tokens"] + prompt["provenance"] = { + "path": f"provenance/{digest}.json", + "sha256": digest, + } + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) diff --git a/tools/deepseek-v41-trace/prompt-builder.cpp b/tools/deepseek-v41-trace/prompt-builder.cpp new file mode 100644 index 000000000000..4f1b7cb63b7e --- /dev/null +++ b/tools/deepseek-v41-trace/prompt-builder.cpp @@ -0,0 +1,459 @@ +#include "build-info.h" +#include "common.h" +#include "ggml-backend.h" +#include "ggml.h" +#include "host-attestation.h" +#include "dsv41-runtime-receipt.h" +extern "C" { +#include "hash/sha256/sha256.h" +} +#include "llama.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#include +#endif + +namespace fs = std::filesystem; +using json = nlohmann::ordered_json; + +#if defined(__linux__) +static constexpr const char * BUILD_REVISION = DSV41_BUILD_REVISION; +#endif + +static std::string sha256_hex(const unsigned char digest[SHA256_DIGEST_SIZE]) { + std::ostringstream stream; + stream << std::hex << std::setfill('0'); + for (size_t i = 0; i < SHA256_DIGEST_SIZE; ++i) { + stream << std::setw(2) << static_cast(digest[i]); + } + return stream.str(); +} + +static std::string sha256_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open for SHA-256: " + path.string()); + } + sha256_t state; + sha256_init(&state); + std::vector buffer(1024 * 1024); + while (input) { + input.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + const std::streamsize count = input.gcount(); + if (count > 0) { + sha256_update(&state, buffer.data(), static_cast(count)); + } + } + if (!input.eof()) { + throw std::runtime_error("failed while hashing: " + path.string()); + } + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_final(&state, digest); + return sha256_hex(digest); +} + +static fs::path canonical_path(const fs::path & path, const char * label) { + try { + return fs::canonical(path); + } catch (const fs::filesystem_error & error) { + throw std::runtime_error(std::string("cannot resolve ") + label + ": " + error.what()); + } +} + +#if defined(__linux__) +static fs::path current_executable_path() { + return canonical_path("/proc/self/exe", "current executable"); +} + +template +static const void * function_address(T function) { + return reinterpret_cast(reinterpret_cast(function)); +} + +static fs::path module_path(const void * address) { + Dl_info info = {}; + if (dladdr(address, &info) == 0 || info.dli_fname == nullptr || info.dli_fname[0] == '\0') { + throw std::runtime_error("cannot identify loaded runtime module"); + } + return canonical_path(info.dli_fname, "loaded runtime module"); +} + +static bool is_project_runtime_library(const fs::path & path) { + std::string name = path.filename().string(); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { + return static_cast(std::tolower(value)); + }); + return name.rfind("libllama", 0) == 0 || name.rfind("libggml", 0) == 0 || + name.rfind("ggml", 0) == 0; +} + +static bool is_trusted_system_runtime_path(const fs::path & path) { + const std::string value = path.string(); + std::error_code error; + const fs::path rocm_root = fs::canonical("/opt/rocm", error); + if (!error) { + const std::string prefix = rocm_root.string() + "/"; + if (value.rfind(prefix, 0) == 0) { + for (fs::path current = path; current != rocm_root.parent_path(); current = current.parent_path()) { + struct stat status = {}; + if (::stat(current.c_str(), &status) != 0 || status.st_uid != 0 || + (status.st_mode & (S_IWGRP | S_IWOTH)) != 0) { + return false; + } + if (current == rocm_root) { + return true; + } + } + } + } + return value.rfind("/lib/", 0) == 0 || + value.rfind("/lib64/", 0) == 0 || + value.rfind("/usr/lib/", 0) == 0 || + value.rfind("/usr/lib64/", 0) == 0; +} + +static std::set loaded_runtime_images(const fs::path & executable) { + std::set result; + struct image_context { + const fs::path * executable; + std::set * result; + std::string error; + } context = {&executable, &result, {}}; + const auto callback = [](dl_phdr_info * info, size_t, void * data) { + image_context & context = *static_cast(data); + if (!context.error.empty() || info->dlpi_name == nullptr || info->dlpi_name[0] == '\0') { + return 0; + } + try { + const fs::path reported_path = info->dlpi_name; + if (!reported_path.is_absolute() && + (reported_path == "linux-vdso.so.1" || reported_path == "linux-gate.so.1")) { + return 0; + } + const fs::path path = canonical_path(reported_path, "loaded runtime module"); + if (path != *context.executable) { + context.result->insert(path); + } + } catch (const std::exception & error) { + context.error = error.what(); + } + return 0; + }; + if (dl_iterate_phdr(callback, &context) < 0 || !context.error.empty()) { + throw std::runtime_error( + context.error.empty() ? "cannot enumerate loaded runtime modules" : context.error); + } + return result; +} + +static fs::path runtime_library_directory(const fs::path & executable) { + if (std::string(dsv41_runtime_receipt::profile) != "sibling-lib") { + throw std::runtime_error("prompt builder requires the sibling-lib runtime profile"); + } + return executable.parent_path().parent_path() / "lib"; +} + +static void load_runtime_backends(const fs::path & executable) { +#if defined(GGML_BACKEND_DL) + const std::string directory = runtime_library_directory(executable).string(); + ggml_backend_load_all_from_path(directory.c_str()); +#else + (void) executable; + ggml_backend_load_all(); +#endif +} + +static json runtime_libraries_json(const fs::path & executable) { + if (dsv41_runtime_receipt::entries.size() != dsv41_runtime_receipt::components.size()) { + throw std::runtime_error("embedded runtime profile size differs from the receipt"); + } + const fs::path library_directory = + canonical_path(runtime_library_directory(executable), "runtime library directory"); + std::map receipt_by_component; + std::map receipt_by_path; + for (size_t index = 0; index < dsv41_runtime_receipt::entries.size(); ++index) { + const dsv41_runtime_receipt::entry & entry = dsv41_runtime_receipt::entries[index]; + if (std::string(entry.component) != dsv41_runtime_receipt::components[index]) { + throw std::runtime_error("embedded runtime profile differs from the receipt"); + } + const fs::path expected = + canonical_path(library_directory / entry.filename, "receipt runtime module"); + if (!receipt_by_component.emplace(entry.component, &entry).second || + !receipt_by_path.emplace(expected, &entry).second) { + throw std::runtime_error("embedded runtime receipt is not canonical and unique"); + } + } + std::set libraries; + for (const fs::path & image : loaded_runtime_images(executable)) { + if (image.parent_path() == library_directory || is_project_runtime_library(image)) { + libraries.insert(image); + } else if (!is_trusted_system_runtime_path(image)) { + throw std::runtime_error("loaded unclassified module outside trusted system roots: " + image.string()); + } + } + std::map loaded_by_component; + for (const fs::path & library : libraries) { + const auto receipt = receipt_by_path.find(library); + if (library.parent_path() != library_directory || receipt == receipt_by_path.end()) { + throw std::runtime_error("loaded project runtime module is absent from the receipt: " + library.string()); + } + if (!loaded_by_component.emplace(receipt->second->component, library).second || + sha256_file(library) != receipt->second->sha256) { + throw std::runtime_error("loaded runtime module identity differs from the receipt"); + } + } + if (loaded_by_component.size() != receipt_by_component.size()) { + throw std::runtime_error("loaded runtime component set differs from the receipt"); + } + const std::array, 3> fixed_roles = {{ + {"llama-common", module_path(function_address(&llama_commit))}, + {"llama", module_path(function_address(&llama_model_load_from_file))}, + {"ggml-base", module_path(function_address(&ggml_init))}, + }}; + for (const auto & item : fixed_roles) { + const auto loaded = loaded_by_component.find(item.first); + if (loaded == loaded_by_component.end() || loaded->second != item.second) { + throw std::runtime_error( + "runtime symbol provider does not match receipt component " + std::string(item.first)); + } + } + json result = json::array(); + for (const fs::path & library : libraries) { + const dsv41_runtime_receipt::entry & receipt = *receipt_by_path.at(library); + std::string role = "runtime:" + std::string(receipt.component); + if (library == fixed_roles[0].second) { + role = "build-info"; + } else if (library == fixed_roles[1].second) { + role = "llama"; + } else if (library == fixed_roles[2].second) { + role = "ggml"; + } + result.push_back({ + {"component", receipt.component}, + {"filename", receipt.filename}, + {"path", library.string()}, + {"sha256", receipt.sha256}, + {"role", std::move(role)}, + {"revision", receipt.revision[0] == '\0' ? json(nullptr) : json(receipt.revision)}, + }); + } + return result; +} + +static json runtime_build_json(const fs::path & executable) { + if (std::string(BUILD_REVISION) != llama_commit() || std::string(BUILD_REVISION) != ggml_commit()) { + throw std::runtime_error("loaded runtime library revision differs from the prompt builder revision"); + } + const json runtime_libraries = runtime_libraries_json(executable); + return { + {"revision", BUILD_REVISION}, + {"path", executable.string()}, + {"sha256", sha256_file(executable)}, + {"runtime_profile", { + {"name", dsv41_runtime_receipt::profile}, + {"components", dsv41_runtime_receipt::components}, + {"selected_backend_component", "ggml-hip"}, + }}, + {"runtime_receipt_sha256", dsv41_runtime_receipt::sha256}, + {"runtime_libraries", runtime_libraries}, + {"runtime_libraries_post", runtime_libraries}, + }; +} +#endif + +static std::string read_file(const fs::path & path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open: " + path.string()); + } + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +static std::string argument(int argc, char ** argv, const std::string & name) { + for (int index = 1; index + 1 < argc; ++index) { + if (argv[index] == name) { + return argv[index + 1]; + } + } + throw std::runtime_error("missing argument: " + name); +} + +static bool boolean_argument(int argc, char ** argv, const std::string & name) { + const std::string value = argument(argc, argv, name); + if (value == "true") { + return true; + } + if (value == "false") { + return false; + } + throw std::runtime_error(name + " must be true or false"); +} + +static std::string model_architecture(const llama_model * model) { + char buffer[128] = {}; + if (llama_model_meta_val_str(model, "general.architecture", buffer, sizeof(buffer)) < 0) { + throw std::runtime_error("model has no general.architecture metadata"); + } + return buffer; +} + +int main(int argc, char ** argv) { + try { +#if !defined(__linux__) + (void) argc; + (void) argv; + throw std::runtime_error("prompt builder runtime attestation requires Linux"); +#else + if (argc == 2 && std::string(argv[1]) == "--dsv41-attest-build") { + const fs::path executable_path = current_executable_path(); + const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked prompt builder"); + if (invoked_path != executable_path) { + throw std::runtime_error("invoked prompt builder path does not match the running executable"); + } + llama_backend_init(); + load_runtime_backends(executable_path); + std::printf("%s\n", runtime_build_json(executable_path).dump().c_str()); + return 0; + } + fs::path model_path = argument(argc, argv, "--model"); + fs::path corpus_path = argument(argc, argv, "--corpus"); + fs::path output_path = argument(argc, argv, "--output"); + const int64_t target_tokens = std::stoll(argument(argc, argv, "--tokens")); + const bool expected_add_bos = boolean_argument(argc, argv, "--tokenizer-add-bos"); + const bool parse_special = boolean_argument(argc, argv, "--tokenizer-parse-special"); + const bool detokenize_special = boolean_argument(argc, argv, "--tokenizer-detokenize-special"); + const bool remove_leading_bos = boolean_argument(argc, argv, "--tokenizer-remove-leading-bos"); + const bool require_round_trip = boolean_argument(argc, argv, "--tokenizer-require-round-trip"); + if (target_tokens < 2) { + throw std::runtime_error("--tokens must be at least 2"); + } + if (!parse_special || !detokenize_special || !require_round_trip || + remove_leading_bos != expected_add_bos) { + throw std::runtime_error("prompt builder requires the exact approved tokenizer policy"); + } + const fs::path executable_path = current_executable_path(); + const fs::path invoked_path = canonical_path(fs::absolute(argv[0]), "invoked prompt builder"); + if (invoked_path != executable_path) { + throw std::runtime_error("invoked prompt builder path does not match the running executable"); + } + llama_backend_init(); + load_runtime_backends(executable_path); + const json runtime_build = runtime_build_json(executable_path); + model_path = dsv41::require_nvme_path(model_path, "model").resolved_path; + corpus_path = dsv41::require_nvme_path(corpus_path, "corpus").resolved_path; + output_path = dsv41::require_nvme_path(output_path, "prompt output").resolved_path; + const char * tmpdir_value = std::getenv("TMPDIR"); + if (tmpdir_value == nullptr || *tmpdir_value == '\0') { + throw std::runtime_error("TMPDIR is required"); + } + dsv41::require_usable_directory(tmpdir_value, "TMPDIR"); + const dsv41::storage_attestation temporary_storage = + dsv41::require_nvme_path(tmpdir_value, "temporary directory"); + if (fs::exists(output_path)) { + throw std::runtime_error("prompt output already exists: " + output_path.string()); + } + + const std::string corpus = read_file(corpus_path); + if (corpus.empty()) { + throw std::runtime_error("corpus is empty"); + } + + llama_model_params model_params = llama_model_default_params(); + model_params.vocab_only = true; + llama_model * model = llama_model_load_from_file(model_path.string().c_str(), model_params); + if (model == nullptr) { + throw std::runtime_error("cannot load model vocabulary"); + } + if (model_architecture(model) != "deepseek41") { + llama_model_free(model); + throw std::runtime_error("prompt builder requires general.architecture=deepseek41"); + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool add_bos = llama_vocab_get_add_bos(vocab); + if (add_bos != expected_add_bos) { + llama_model_free(model); + throw std::runtime_error("model tokenizer add_bos differs from the approved policy"); + } + + std::string repeated = corpus; + std::vector tokens = common_tokenize(vocab, repeated, add_bos, parse_special); + while (tokens.size() < static_cast(target_tokens)) { + if (repeated.size() > (size_t(1) << 31)) { + llama_model_free(model); + throw std::runtime_error("repeated prompt exceeds 2 GiB"); + } + repeated += repeated; + tokens = common_tokenize(vocab, repeated, add_bos, parse_special); + } + tokens.resize(static_cast(target_tokens)); + std::vector content_tokens = tokens; + if (remove_leading_bos) { + if (content_tokens.front() != llama_vocab_bos(vocab)) { + llama_model_free(model); + throw std::runtime_error("tokenized prompt does not begin with the configured BOS token"); + } + content_tokens.erase(content_tokens.begin()); + } + const std::string prompt = common_detokenize(vocab, content_tokens, detokenize_special); + const std::vector verified = common_tokenize(vocab, prompt, add_bos, parse_special); + if (require_round_trip && verified != tokens) { + llama_model_free(model); + throw std::runtime_error("constructed prompt does not round-trip to the target token IDs"); + } + + if (!output_path.parent_path().empty()) { + fs::create_directories(output_path.parent_path()); + } + std::ofstream output(output_path, std::ios::binary | std::ios::trunc); + if (!output || !output.write(prompt.data(), static_cast(prompt.size()))) { + llama_model_free(model); + throw std::runtime_error("cannot write prompt output"); + } + output.close(); + const json runtime_build_post = runtime_build_json(executable_path); + if (runtime_build_post != runtime_build) { + llama_model_free(model); + throw std::runtime_error("loaded runtime component set changed during prompt construction"); + } + std::printf("%s\n", json({ + {"target_tokens", target_tokens}, + {"actual_tokens", verified.size()}, + {"byte_count", prompt.size()}, + {"tokenizer", { + {"add_bos", expected_add_bos}, + {"parse_special", parse_special}, + {"detokenize_special", detokenize_special}, + {"remove_leading_bos_before_detokenize", remove_leading_bos}, + {"require_round_trip", require_round_trip}, + }}, + {"runtime_build", runtime_build_post}, + {"temporary_directory", temporary_storage.resolved_path.string()}, + }).dump().c_str()); + llama_model_free(model); + return 0; +#endif + } catch (const std::exception & error) { + std::fprintf(stderr, "error: %s\n", error.what()); + return 1; + } +} diff --git a/tools/deepseek-v41-trace/run_ds4.py b/tools/deepseek-v41-trace/run_ds4.py new file mode 100644 index 000000000000..bd6ea82d53fa --- /dev/null +++ b/tools/deepseek-v41-trace/run_ds4.py @@ -0,0 +1,804 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import shlex +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from preflight import ( + PreflightError, + bind_embedded_audits, + bind_prompt_provenance, + darwin_storage_attestation, + resolved, + run_oracle_preflight, + seal_audits, + validate_prompt_provenance, + verify_sealed_audits, + write_audits, +) +from trace_format import ( + ADMITTED_UBATCH, + APPROVED_PROMPT_BUILDERS, + APPROVED_TRACE_SIGNERS, + CORPUS_SHA256, + DS4_REPOSITORY, + DS4_REVISION, + ExecutionIntegrityError, + ExecutableFileReceipt, + MODEL_SHA256, + NO_EXTERNAL_STATE_STORAGE, + ORACLE_LANE, + TraceBundle, + TraceError, + TraceVerifier, + approval_binding, + approved_executable_identity, + approved_runtime_file_identities, + bind_execution_authorization, + canonical_json, + ds4_exporter_approval, + execution_authorization, + install_trust_evidence, + install_trust_sha256, + load_executable_approval_policy, + prompt_builder_approval, + reject_loader_overrides, + run_approved_executable, + runtime_build_evidence_sha256, + seal_bundle, + sha256_bytes, + sha256_file, + strict_json_loads, + tokenizer_policy_sha256, + validate_runtime_build_evidence, + validate_signing_identity, + verify_approved_executable_identity, + verify_approved_runtime_file_identities, +) + +EXPORTER_ATTESTATION_TIMEOUT_SECONDS = 60 +EXPORTER_TRACE_TIMEOUT_SECONDS = 24 * 60 * 60 + + +@dataclass(frozen=True) +class InvocationSecondaryFailure: + component: str + error: BaseException + + +class InvocationIntegrityError(PreflightError): + def __init__( + self, + message: str, + *, + primary_error: BaseException, + secondary_errors: list[InvocationSecondaryFailure]): + super().__init__(message) + self.primary_error = primary_error + self.secondary_errors = tuple(secondary_errors) + + +def git_output(checkout: Path, *args: str) -> str: + try: + return subprocess.check_output( + ["git", "-C", str(checkout), *args], + text=True, + stderr=subprocess.STDOUT, + ).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"ds4 git {' '.join(args)} failed: {error}") from error + + +def verify_checkout(checkout: Path) -> str: + revision = git_output(checkout, "rev-parse", "HEAD") + status = git_output(checkout, "status", "--porcelain", "--untracked-files=all") + if status: + raise PreflightError("ds4 checkout has tracked or untracked changes") + return revision + + +def validate_accelerator_attestation( + record: object, + *, + expected_device: str = "Metal0") -> dict[str, object]: + if not isinstance(record, dict): + raise PreflightError("accelerator attestation is not an object") + required_keys = { + "format", + "version", + "runtime_kind", + "platform", + "backend", + "backend_device", + "backend_description", + "architecture", + "metal_registry_id", + "recommended_max_working_set_bytes", + "unified_memory", + "source", + } + if set(record) != required_keys: + raise PreflightError("accelerator attestation fields are invalid") + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "backend": "Metal", + "backend_device": expected_device, + "unified_memory": True, + "source": "metal-device-query", + } + for key, value in expected.items(): + if record.get(key) != value: + raise PreflightError(f"accelerator attestation {key} mismatch") + if type(record.get("unified_memory")) is not bool: + raise PreflightError("accelerator attestation unified-memory identity is invalid") + for key in ("backend_description", "architecture"): + if not isinstance(record.get(key), str) or not record[key]: + raise PreflightError(f"accelerator attestation {key} is missing") + if type(record.get("metal_registry_id")) is not int or record["metal_registry_id"] <= 0: + raise PreflightError("accelerator attestation Metal registry identity is invalid") + if type(record.get("recommended_max_working_set_bytes")) is not int or ( + record["recommended_max_working_set_bytes"] <= 0): + raise PreflightError("accelerator attestation working-set identity is invalid") + return dict(record) + + +def run_exporter_command( + command: list[str], + *, + exporter: Path, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any], + timeout_seconds: int | None = None, + **kwargs: Any) -> subprocess.CompletedProcess[Any]: + if type(timeout_seconds) is not int or timeout_seconds <= 0 or ( + {"timeout", "text", "encoding", "errors", "universal_newlines"} & kwargs.keys()): + raise PreflightError("ds4 exporter timeout is invalid") + result, executed_identity = run_approved_executable( + command, + path=exporter, + runtime_policy=exporter_policy, + expected_path=exporter_policy["executable_path"], + expected_sha256=exporter_policy["executable_sha256"], + label="ds4 exporter", + timeout=timeout_seconds, + **kwargs, + ) + if executed_identity != exporter_identity: + raise PreflightError("ds4 exporter execution identity differs from external approval") + return result + + +def decode_exporter_output(value: bytes | None, *, label: str) -> str: + if not isinstance(value, bytes): + raise PreflightError(f"{label} bytes are missing") + try: + return value.decode("utf-8", "strict") + except UnicodeError as error: + raise PreflightError(f"{label} is not valid UTF-8: {error}") from error + + +def query_runtime_build_attestation( + exporter: Path, + *, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any]) -> dict[str, Any]: + result = run_exporter_command( + [str(exporter), "--dsv41-attest-build"], + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, + check=False, + capture_output=True, + ) + if result.returncode != 0: + detail = decode_exporter_output( + result.stderr, label="ds4 exporter build attestation stderr").strip() + raise PreflightError(f"ds4 exporter build attestation failed: {detail}") + try: + record = strict_json_loads(decode_exporter_output( + result.stdout, label="ds4 exporter build attestation stdout")) + return validate_runtime_build_evidence(record, exporter_policy, label="ds4 exporter") + except TraceError as error: + raise PreflightError(f"ds4 exporter build attestation is invalid: {error}") from error + + +def run_exporter_with_post_attestation( + command: list[str], + *, + operation: str, + exporter: Path, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any], + expected_runtime_build: dict[str, Any], + timeout_seconds: int, + decode_stdout_label: str | None = None, + decode_stderr_label: str | None = None, + **kwargs: Any) -> subprocess.CompletedProcess[Any]: + result = None + primary_error = None + try: + result = run_exporter_command( + command, + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + timeout_seconds=timeout_seconds, + **kwargs, + ) + except BaseException as error: + primary_error = error + if primary_error is not None and ( + not isinstance(primary_error, ExecutionIntegrityError) + or not primary_error.quiescence_proven): + raise primary_error + if result is not None and primary_error is None: + try: + stdout = ( + decode_exporter_output(result.stdout, label=decode_stdout_label) + if decode_stdout_label is not None else result.stdout) + stderr = ( + decode_exporter_output(result.stderr, label=decode_stderr_label) + if decode_stderr_label is not None else result.stderr) + result = subprocess.CompletedProcess( + result.args, result.returncode, stdout, stderr) + except BaseException as error: + primary_error = error + nonzero_error = None + if result is not None and result.returncode != 0: + nonzero_error = PreflightError(f"{operation} failed: exit {result.returncode}") + secondary_error = None + try: + post_runtime_build = query_runtime_build_attestation( + exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + ) + if post_runtime_build != expected_runtime_build: + raise PreflightError(f"ds4 exporter build identity changed during {operation}") + except BaseException as error: + secondary_error = error + reported_primary = primary_error or nonzero_error + if reported_primary is not None: + if secondary_error is not None: + raise InvocationIntegrityError( + f"{operation} primary failure [{type(reported_primary).__name__}: {reported_primary}]; " + f"secondary post-invocation runtime-build attestation failure " + f"[{type(secondary_error).__name__}: {secondary_error}]", + primary_error=reported_primary, + secondary_errors=[InvocationSecondaryFailure( + "post-invocation-runtime-build-attestation", secondary_error)], + ) from reported_primary + if primary_error is not None: + raise primary_error + if secondary_error is not None: + raise secondary_error + if result is None: + raise PreflightError(f"{operation} did not return a result") + return result + + +def query_accelerator_attestation( + exporter: Path, + device: str, + *, + exporter_identity: ExecutableFileReceipt, + exporter_policy: dict[str, Any], + expected_runtime_build: dict[str, Any]) -> dict[str, object]: + result = run_exporter_with_post_attestation( + [str(exporter), "--dsv41-attest-device", device], + operation="selected accelerator query", + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + expected_runtime_build=expected_runtime_build, + timeout_seconds=EXPORTER_ATTESTATION_TIMEOUT_SECONDS, + check=False, + capture_output=True, + decode_stdout_label="selected accelerator query stdout", + decode_stderr_label="selected accelerator query stderr", + ) + validation_error = None + attestation = None + if result.returncode != 0: + detail = result.stderr.strip() or f"exit {result.returncode}" + validation_error = PreflightError(f"selected accelerator query failed: {detail}") + else: + try: + record = strict_json_loads(result.stdout) + attestation = validate_accelerator_attestation(record, expected_device=device) + except (TraceError, PreflightError) as error: + validation_error = PreflightError( + f"selected accelerator query returned invalid attestation: {error}") + if validation_error is not None: + raise validation_error + if attestation is None: + raise PreflightError("selected accelerator attestation is missing") + return attestation + + +def runner_attestation( + *, + exporter: Path, + exporter_sha256: str, + exporter_approval_id: str, + exporter_approval_sha256: str, + exporter_install_trust_sha256: str, + exporter_runtime_build_sha256: str, + exporter_runtime_profile: dict[str, object], + exporter_runtime_receipt_sha256: str, + verifier_revision: str, + checkout: Path, + command: list[str]) -> dict[str, object]: + runner_executable = resolved(Path(sys.executable)) + runner_script = resolved(Path(__file__)) + return { + "format": "dsv41-runner-ownership", + "version": 1, + "runtime_kind": "apple-metal", + "source": "python-subprocess", + "runner_pid": os.getpid(), + "runner_parent_pid": os.getppid(), + "runner_uid": os.getuid(), + "runner_executable": str(runner_executable), + "runner_executable_sha256": sha256_file(runner_executable), + "runner_script": str(runner_script), + "runner_script_sha256": sha256_file(runner_script), + "exporter_path": str(exporter), + "exporter_sha256": exporter_sha256, + "exporter_approval_id": exporter_approval_id, + "exporter_approval_sha256": exporter_approval_sha256, + "exporter_install_trust_sha256": exporter_install_trust_sha256, + "exporter_runtime_build_sha256": exporter_runtime_build_sha256, + "exporter_runtime_profile": exporter_runtime_profile, + "exporter_runtime_receipt_sha256": exporter_runtime_receipt_sha256, + "producer_revision": DS4_REVISION, + "verifier_revision": verifier_revision, + "checkout_path": str(checkout), + "checkout_revision": DS4_REVISION, + "command_sha256": sha256_bytes(canonical_json(command).encode("ascii")), + } + + +def bind_oracle_attestation( + output: Path, + audit: dict[str, object], + accelerator: dict[str, object], + command: list[str], + *, + exporter_policy: dict[str, object], + exporter_approval_id: str, + exporter_approval_sha256: str, + exporter_install_trust: dict[str, object], + runtime_build: dict[str, object], + verifier_revision: str) -> None: + manifest_path = output / "manifest.json" + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: + raise PreflightError(f"cannot bind ds4 runtime attestation: {error}") from error + if manifest.get("accelerator") != accelerator: + raise PreflightError("ds4 trace accelerator attestation differs from the preflight query") + storage = audit.get("storage") + if not isinstance(storage, dict): + raise PreflightError("ds4 storage attestation is missing") + paths = {} + for label, record in storage.items(): + if not isinstance(record, dict) or not isinstance(record.get("resolved_path"), str): + raise PreflightError(f"ds4 storage attestation is invalid for {label}") + paths[label] = record["resolved_path"] + if manifest.get("model", {}).get("path") != paths["model"]: + raise PreflightError("ds4 trace model path differs from the attested path") + if manifest.get("prompt", {}).get("path") != paths["prompt"]: + raise PreflightError("ds4 trace prompt path differs from the attested path") + if "paths" in manifest and manifest["paths"] != paths: + raise PreflightError("ds4 trace execution paths differ from preflight") + manifest["paths"] = paths + build = manifest.get("build") + if not isinstance(build, dict) or build.get("path") != paths["exporter"]: + raise PreflightError("ds4 trace build path differs from the executed exporter") + runner = audit.get("runner") + if not isinstance(runner, dict) or build.get("sha256") != runner.get("exporter_sha256"): + raise PreflightError("ds4 trace build SHA-256 differs from the executed exporter") + build_evidence = { + "revision": manifest.get("revision"), + "path": build.get("path"), + "sha256": build.get("sha256"), + "runtime_profile": build.get("runtime_profile"), + "runtime_receipt_sha256": build.get("runtime_receipt_sha256"), + "runtime_libraries": build.get("runtime_libraries"), + "runtime_libraries_post": build.get("runtime_libraries_post"), + } + try: + validated_build = validate_runtime_build_evidence( + build_evidence, exporter_policy, label="ds4 exporter") + except TraceError as error: + raise PreflightError(f"ds4 trace runtime build differs from external approval: {error}") from error + if validated_build != runtime_build: + raise PreflightError("ds4 trace runtime build differs from measured exporter attestation") + runtime_build_sha256 = runtime_build_evidence_sha256( + validated_build, exporter_policy, label="ds4 exporter") + runtime_libraries_sha256 = sha256_bytes(canonical_json({ + "pre": validated_build["runtime_libraries"], + "post": validated_build["runtime_libraries_post"], + }).encode("ascii")) + runtime_receipt_sha256 = sha256_bytes( + canonical_json(exporter_policy["runtime_receipt"]).encode("ascii")) + trust_sha256 = install_trust_sha256(exporter_install_trust) + manifest["oracle"] = { + "repository": DS4_REPOSITORY, + "revision": DS4_REVISION, + "verifier_revision": verifier_revision, + "executable_path": exporter_policy["executable_path"], + "executable_sha256": exporter_policy["executable_sha256"], + "runtime_profile": exporter_policy["runtime_profile"], + "runtime_build_sha256": runtime_build_sha256, + "runtime_libraries_sha256": runtime_libraries_sha256, + "runtime_receipt_sha256": runtime_receipt_sha256, + "exporter_approval_id": exporter_approval_id, + "exporter_approval_sha256": exporter_approval_sha256, + "install_trust": exporter_install_trust, + "install_trust_sha256": trust_sha256, + } + storage_policy = audit.get("storage_policy") + if storage_policy != NO_EXTERNAL_STATE_STORAGE: + raise PreflightError("ds4 external cache/state storage policy is invalid") + if "storage_policy" in manifest and manifest["storage_policy"] != storage_policy: + raise PreflightError("ds4 trace external cache/state storage policy differs from preflight") + manifest["storage_policy"] = storage_policy + host = audit.get("host") + if not isinstance(host, dict): + raise PreflightError("ds4 host attestation is missing") + if "host" in manifest and manifest["host"] != host: + raise PreflightError("ds4 trace host identity differs from preflight") + manifest["host"] = host + manifest["environment"] = { + "system_info": f"macOS {host['os_version']} arm64 {host['hardware_model']}", + "command": shlex.join(command), + } + config = manifest.get("config") + if not isinstance(config, dict): + raise PreflightError("ds4 trace config is invalid") + for key, value in ( + ("device_backend", "Metal"), + ("device_registry_id", accelerator["metal_registry_id"])): + if key in config and config[key] != value: + raise PreflightError(f"ds4 trace {key} differs from preflight") + config[key] = value + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +def preflight( + args: argparse.Namespace, + *, + accelerator: dict[str, object], + runner: dict[str, object]) -> dict[str, object]: + checkout = resolved(args.checkout) + revision = verify_checkout(checkout) + if revision != DS4_REVISION: + raise PreflightError(f"ds4 revision mismatch: expected {DS4_REVISION}, found {revision}") + result = run_oracle_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + repo=args.repo, + checkout=checkout, + busy_patterns=args.busy_pattern, + accelerator=accelerator, + runner=runner, + ) + result.update({ + "runtime": "ds4", + "ds4_revision": revision, + "checkout": str(checkout), + "config": { + "context": args.context, + "decode_steps": args.decode_steps, + "prefill_chunk": args.prefill_chunk, + "device_backend": "Metal", + "device_registry_id": accelerator["metal_registry_id"], + }, + }) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed launcher for the pinned ds4 trace exporter") + parser.add_argument("--checkout", type=Path, default=Path("/home/papa/src/ds4-v41")) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--prompt", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) + parser.add_argument("--exporter", type=Path, required=True) + parser.add_argument("--exporter-sha256", required=True) + parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) + parser.add_argument("--corpus-sha256", required=True) + parser.add_argument("--prompt-provenance", type=Path, required=True) + parser.add_argument("--context", type=int, default=32768) + parser.add_argument("--decode-steps", type=int, default=8) + parser.add_argument("--prefill-chunk", type=int, default=ADMITTED_UBATCH) + parser.add_argument("--device", default="Metal0") + parser.add_argument("--signer-principal", required=True) + parser.add_argument("--signing-key", type=Path, required=True) + parser.add_argument("--execution-challenge", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--authorization-issued-unix", type=int, required=True) + parser.add_argument("--authorization-expires-unix", type=int, required=True) + parser.add_argument("--ds4-exporter-policy-id", required=True) + parser.add_argument("--prompt-builder-policy-id", required=True) + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) + parser.add_argument("--preflight-only", action="store_true") + args = parser.parse_args() + + try: + if args.prefill_chunk != ADMITTED_UBATCH: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require admitted prefill chunk {ADMITTED_UBATCH}, " + f"found {args.prefill_chunk}") + reject_loader_overrides() + output = resolved(args.output) + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(output,), + ) + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + args.prompt_builder_policy_id, + policies=approval_policy.prompt_builders, + ) + exporter_policy, exporter_policy_sha256 = ds4_exporter_approval( + args.ds4_exporter_policy_id, + policies=approval_policy.ds4_exporters, + ) + harness_repo = resolved(args.repo) + harness_revision = subprocess.check_output( + ["git", "-C", str(harness_repo), "rev-parse", "HEAD"], + stderr=subprocess.STDOUT, + ).decode("ascii").strip() + if harness_revision != approval_policy.verifier_revision: + raise PreflightError("ds4 verifier checkout differs from the external approval policy") + checkout = resolved(args.checkout) + checkout_revision = verify_checkout(checkout) + if checkout_revision != DS4_REVISION: + raise PreflightError( + f"ds4 revision mismatch: expected {DS4_REVISION}, found {checkout_revision}") + if not args.exporter.is_absolute() or str(args.exporter) != exporter_policy["executable_path"]: + raise PreflightError("ds4 exporter path or caller digest differs from external approval") + exporter = resolved(args.exporter) + if str(exporter) != str(args.exporter) or ( + args.exporter_sha256 != exporter_policy["executable_sha256"]): + raise PreflightError("ds4 exporter path or caller digest differs from external approval") + exporter_identity = approved_executable_identity( + exporter, + install_root=exporter_policy["install_root"], + expected_owner_uid=exporter_policy["install_owner_uid"], + expected_path=exporter_policy["executable_path"], + expected_sha256=exporter_policy["executable_sha256"], + label="ds4 exporter", + ) + runtime_identities = approved_runtime_file_identities( + exporter_policy, label="ds4 exporter") + exporter_install_trust = install_trust_evidence( + exporter_identity, runtime_identities) + exporter_install_trust_sha256 = install_trust_sha256(exporter_install_trust) + pre_runtime_build = query_runtime_build_attestation( + exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + ) + runtime_build_sha256 = runtime_build_evidence_sha256( + pre_runtime_build, exporter_policy, label="ds4 exporter") + runtime_receipt_sha256 = sha256_bytes( + canonical_json(exporter_policy["runtime_receipt"]).encode("ascii")) + if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: + raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") + model_sha256 = sha256_file(resolved(args.model)) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + provenance = validate_prompt_provenance( + args.prompt_provenance, + prompt=args.prompt, + corpus_name=args.corpus_name, + corpus_sha256=args.corpus_sha256, + model_sha256=model_sha256, + target_tokens=args.context - args.decode_steps, + context=args.context, + decode_steps=args.decode_steps, + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, + path_resolver=lambda path, label: Path( + str(darwin_storage_attestation(path, label)["resolved_path"])), + ) + authorization = execution_authorization( + lane=ORACLE_LANE, + challenge=args.execution_challenge, + run_id=args.run_id, + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), + approvals={ + "ds4_exporter": approval_binding( + "ds4_exporter", + args.ds4_exporter_policy_id, + exporter_policy_sha256, + exporter_install_trust_sha256, + ), + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + provenance["record"]["builder_install_trust_sha256"], + ), + }, + ) + exporter_sha256 = exporter_identity.sha256 + validate_signing_identity( + args.signing_key, + args.signer_principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + forbidden_root=output, + ) + command = [ + str(exporter), + "--model", str(resolved(args.model)), + "--prompt-file", str(resolved(args.prompt)), + "--output", str(output), + "--context", str(args.context), + "--decode-steps", str(args.decode_steps), + "--prefill-chunk", str(args.prefill_chunk), + "--device", args.device, + ] + accelerator = query_accelerator_attestation( + exporter, + args.device, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + expected_runtime_build=pre_runtime_build, + ) + runner = runner_attestation( + exporter=exporter, + exporter_sha256=exporter_sha256, + exporter_approval_id=args.ds4_exporter_policy_id, + exporter_approval_sha256=exporter_policy_sha256, + exporter_install_trust_sha256=exporter_install_trust_sha256, + exporter_runtime_build_sha256=runtime_build_sha256, + exporter_runtime_profile=exporter_policy["runtime_profile"], + exporter_runtime_receipt_sha256=runtime_receipt_sha256, + verifier_revision=approval_policy.verifier_revision, + checkout=checkout, + command=command, + ) + if args.preflight_only: + audit = preflight(args, accelerator=accelerator, runner=runner) + print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) + return 0 + + if output.exists() and any(output.iterdir()): + raise PreflightError(f"trace output directory is not empty: {output}") + preflight_audit = preflight(args, accelerator=accelerator, runner=runner) + preflight_audit["exporter"] = { + "path": str(exporter), + "sha256": exporter_sha256, + "approval_id": args.ds4_exporter_policy_id, + "approval_sha256": exporter_policy_sha256, + "install_trust_sha256": exporter_install_trust_sha256, + "runtime_build_sha256": runtime_build_sha256, + "runtime_receipt_sha256": runtime_receipt_sha256, + } + pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) + pre_audit_digests = seal_audits(pre_audits) + print("exec:", shlex.join(command), file=sys.stderr) + result = run_exporter_with_post_attestation( + command, + operation="ds4 trace execution", + exporter=exporter, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + expected_runtime_build=pre_runtime_build, + timeout_seconds=EXPORTER_TRACE_TIMEOUT_SECONDS, + cwd=checkout, + check=False, + ) + verify_approved_executable_identity( + exporter, exporter_identity, label="ds4 exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="ds4 exporter") + if result.returncode != 0: + return result.returncode + verify_sealed_audits(pre_audits, pre_audit_digests) + post_accelerator = query_accelerator_attestation( + exporter, + args.device, + exporter_identity=exporter_identity, + exporter_policy=exporter_policy, + expected_runtime_build=pre_runtime_build, + ) + if post_accelerator != accelerator: + raise PreflightError("selected accelerator identity changed during trace execution") + postflight_audit = preflight(args, accelerator=post_accelerator, runner=runner) + if postflight_audit.get("host") != preflight_audit.get("host"): + raise PreflightError("ds4 host identity changed during trace execution") + post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) + bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) + bind_prompt_provenance(output, provenance) + bind_oracle_attestation( + output, + preflight_audit, + accelerator, + command, + exporter_policy=exporter_policy, + exporter_approval_id=args.ds4_exporter_policy_id, + exporter_approval_sha256=exporter_policy_sha256, + exporter_install_trust=exporter_install_trust, + runtime_build=pre_runtime_build, + verifier_revision=approval_policy.verifier_revision, + ) + bind_execution_authorization(output, authorization) + seal_bundle( + output, + private_key=args.signing_key, + principal=args.signer_principal, + expected_lane=ORACLE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + candidate_exporter_policies={}, + ds4_exporter_policies=approval_policy.ds4_exporters, + prompt_builder_policies=approval_policy.prompt_builders, + expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=args.ds4_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + expected_approval_policy_sha256=approval_policy.sha256, + expected_verifier_revision=approval_policy.verifier_revision, + trusted_signers=APPROVED_TRACE_SIGNERS, + ) + bundle = TraceBundle( + output, + verifier=TraceVerifier.production( + args.signer_principal, + expected_lane=ORACLE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=args.ds4_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + ), + ) + if bundle.manifest.get("runtime") != "ds4": + raise PreflightError("ds4 exporter wrote a non-ds4 trace") + if bundle.manifest.get("revision") != DS4_REVISION: + raise PreflightError( + f"ds4 trace revision mismatch: expected {DS4_REVISION}, found {bundle.manifest.get('revision')}") + if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: + raise PreflightError("ds4 trace build SHA-256 does not match the executed exporter") + if bundle.manifest.get("model", {}).get("sha256") != MODEL_SHA256: + raise PreflightError("ds4 trace model SHA-256 does not match the published GGUF") + if bundle.manifest.get("accelerator") != accelerator: + raise PreflightError("ds4 trace accelerator attestation differs from the measured Metal device") + return 0 + except (PreflightError, TraceError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/run_llama.py b/tools/deepseek-v41-trace/run_llama.py new file mode 100644 index 000000000000..40696c93b1d0 --- /dev/null +++ b/tools/deepseek-v41-trace/run_llama.py @@ -0,0 +1,954 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import hashlib +import json +import os +import re +import shlex +import stat +import subprocess +import sys +from pathlib import Path + +from preflight import ( + PreflightError, + bind_embedded_audits, + bind_prompt_provenance, + open_watchdog_namespace_authority, + resolved, + run_strix_preflight, + safe_trace_path, + seal_audits, + validate_prompt_provenance, + verify_watchdog_namespace_authority, + verify_sealed_audits, + write_audits, +) +from trace_format import ( + ADMITTED_BATCH, + ADMITTED_UBATCH, + APPROVED_CANDIDATE_EXPORTERS, + APPROVED_PROMPT_BUILDERS, + APPROVED_TRACE_SIGNERS, + CANDIDATE_LANE, + CORPUS_SHA256, + MODEL_SHA256, + REPOSITORY, + REQUIRED_EXPERT_CACHE_BYTES, + REQUIRED_EXPERT_CACHE_MIB, + REQUIRED_EXPERT_SLOTS, + TraceBundle, + TraceError, + TraceVerifier, + approval_binding, + approved_containment_helper_identity, + approved_executable_identity, + approved_runtime_file_identities, + bind_execution_authorization, + candidate_exporter_approval, + canonical_json, + execution_authorization, + install_trust_evidence, + install_trust_sha256, + load_executable_approval_policy, + reject_loader_overrides, + run_approved_executable, + seal_bundle, + sha256_bytes, + sha256_file, + strict_json_loads, + tokenizer_policy_sha256, + prompt_builder_approval, + validate_signing_identity, + validate_tokenizer_policy, + verify_approved_executable_identity, + verify_approved_runtime_file_identities, +) + + +def approved_source_root(prompt_policy: dict[str, object]) -> Path: + try: + return Path(str(prompt_policy["source_root"])).expanduser().resolve(strict=True) + except (KeyError, OSError) as error: + raise PreflightError(f"prompt builder approved source root is invalid: {error}") from error + + +def git_output(repo: Path, *args: str) -> bytes: + try: + return subprocess.check_output(["git", "-C", str(repo), *args], stderr=subprocess.STDOUT) + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"git {' '.join(args)} failed: {error}") from error + + +def sha256_descriptor(descriptor: int) -> str: + digest = hashlib.sha256() + offset = 0 + try: + while True: + chunk = os.pread(descriptor, 8 * 1024 * 1024, offset) + if not chunk: + return digest.hexdigest() + digest.update(chunk) + offset += len(chunk) + except AttributeError as error: + raise PreflightError("descriptor hashing requires os.pread") from error + except OSError as error: + raise PreflightError(f"cannot hash held model descriptor: {error}") from error + + +def model_descriptor_identity( + descriptor: int, + path: Path, + *, + hash_bytes: bool, +) -> dict[str, object]: + try: + import fcntl + + record = os.fstat(descriptor) + status_flags = fcntl.fcntl(descriptor, fcntl.F_GETFL) + descriptor_flags = fcntl.fcntl(descriptor, fcntl.F_GETFD) + except (ImportError, OSError) as error: + raise PreflightError(f"cannot inspect held model descriptor: {error}") from error + if not stat.S_ISREG(record.st_mode): + raise PreflightError("model descriptor is not a regular file") + if record.st_nlink < 1: + raise PreflightError("model descriptor has no linked pathname") + if (status_flags & os.O_ACCMODE) != os.O_RDONLY: + raise PreflightError("model descriptor is not read-only") + identity = { + "format": "dsv41-model-file-identity", + "version": 1, + "path": str(path), + "device": record.st_dev, + "inode": record.st_ino, + "owner_uid": record.st_uid, + "owner_gid": record.st_gid, + "mode": stat.S_IMODE(record.st_mode), + "link_count": record.st_nlink, + "byte_count": record.st_size, + "modified_ns": record.st_mtime_ns, + "changed_ns": record.st_ctime_ns, + "status_flags": status_flags, + "source_descriptor_flags": descriptor_flags, + "target_descriptor_flags": 0, + } + if hash_bytes: + identity["sha256"] = sha256_descriptor(descriptor) + return identity + + +def open_model_descriptor(path: Path) -> tuple[int, dict[str, object]]: + lexical_path = path.expanduser() + if not lexical_path.is_absolute(): + lexical_path = Path.cwd() / lexical_path + try: + canonical_path = lexical_path.resolve(strict=True) + except OSError as error: + raise PreflightError(f"cannot resolve model path: {error}") from error + if canonical_path != lexical_path: + raise PreflightError("model path must not contain lexical or symbolic-link aliases") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = -1 + try: + descriptor = os.open(canonical_path, flags) + before = model_descriptor_identity(descriptor, canonical_path, hash_bytes=False) + identity = model_descriptor_identity(descriptor, canonical_path, hash_bytes=True) + after = model_descriptor_identity(descriptor, canonical_path, hash_bytes=False) + if before != after: + raise PreflightError("model descriptor identity changed while hashing") + path_record = canonical_path.stat(follow_symlinks=False) + if ( + path_record.st_dev, + path_record.st_ino, + path_record.st_uid, + path_record.st_gid, + stat.S_IMODE(path_record.st_mode), + path_record.st_nlink, + path_record.st_size, + path_record.st_mtime_ns, + path_record.st_ctime_ns, + ) != ( + identity["device"], + identity["inode"], + identity["owner_uid"], + identity["owner_gid"], + identity["mode"], + identity["link_count"], + identity["byte_count"], + identity["modified_ns"], + identity["changed_ns"], + ): + raise PreflightError("model pathname does not identify the held descriptor") + return descriptor, identity + except BaseException: + if descriptor >= 0: + os.close(descriptor) + raise + + +def verify_model_descriptor( + descriptor: int, + identity: dict[str, object], +) -> None: + path = Path(str(identity["path"])) + observed = model_descriptor_identity(descriptor, path, hash_bytes=True) + if observed != identity: + raise PreflightError("model descriptor identity or bytes changed during execution") + try: + path_record = path.stat(follow_symlinks=False) + except OSError as error: + raise PreflightError(f"cannot revalidate model pathname: {error}") from error + if ( + path_record.st_dev, + path_record.st_ino, + path_record.st_uid, + path_record.st_gid, + stat.S_IMODE(path_record.st_mode), + path_record.st_nlink, + path_record.st_size, + path_record.st_mtime_ns, + path_record.st_ctime_ns, + ) != ( + identity["device"], + identity["inode"], + identity["owner_uid"], + identity["owner_gid"], + identity["mode"], + identity["link_count"], + identity["byte_count"], + identity["modified_ns"], + identity["changed_ns"], + ): + raise PreflightError("model pathname identity changed during execution") + + +def candidate_attestation( + args: argparse.Namespace, + exporter: Path, + exporter_sha256: str, + approval_id: str, + approval_sha256: str, + approval: dict[str, object], + verifier_revision: str, + install_trust: dict[str, object]) -> dict[str, object]: + repo = resolved(args.repo) + exporter = resolved(exporter) + observed_verifier_revision = git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() + revision = git_output(repo, "rev-parse", args.candidate_revision).decode("ascii").strip() + base_revision = git_output(repo, "rev-parse", args.base_revision).decode("ascii").strip() + if observed_verifier_revision != verifier_revision: + raise PreflightError( + f"verifier revision mismatch: expected {verifier_revision}, found {observed_verifier_revision}") + if revision != args.candidate_revision: + raise PreflightError( + f"candidate revision mismatch: expected {args.candidate_revision}, found {revision}") + if base_revision != args.base_revision: + raise PreflightError( + f"base revision mismatch: expected {args.base_revision}, found {base_revision}") + try: + subprocess.run( + ["git", "-C", str(repo), "merge-base", "--is-ancestor", base_revision, revision], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["git", "-C", str(repo), "diff", "--quiet"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["git", "-C", str(repo), "diff", "--cached", "--quiet"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + except (OSError, subprocess.CalledProcessError) as error: + raise PreflightError(f"candidate repository is not cleanly based on {base_revision}: {error}") from error + status = git_output(repo, "status", "--porcelain", "--untracked-files=all") + if status: + raise PreflightError("candidate repository has tracked or untracked changes") + diff = git_output(repo, "diff", "--binary", "--no-ext-diff", base_revision, revision, "--") + diff_sha256 = hashlib.sha256(diff).hexdigest() + if diff_sha256 != args.candidate_diff_sha256: + raise PreflightError( + f"candidate diff SHA-256 mismatch: expected {args.candidate_diff_sha256}, found {diff_sha256}") + expected = { + "repository": REPOSITORY, + "revision": revision, + "base_revision": base_revision, + "diff_sha256": diff_sha256, + "executable_path": str(exporter), + "executable_sha256": exporter_sha256, + } + for key, value in expected.items(): + if approval.get(key) != value: + raise PreflightError(f"candidate {key} differs from external exporter approval") + return { + **expected, + "exporter_approval_id": approval_id, + "exporter_approval_sha256": approval_sha256, + "install_trust": install_trust, + "install_trust_sha256": install_trust_sha256(install_trust), + } + + +def validate_runtime_build( + manifest: dict[str, object], + *, + exporter: Path, + exporter_sha256: str, + candidate_revision: str, + approval: dict[str, object]) -> tuple[str, str]: + build = manifest.get("build") + if not isinstance(build, dict): + raise PreflightError("llama trace build identity is missing") + exporter = resolved(exporter) + if build.get("path") != str(exporter): + raise PreflightError("llama trace build path does not match the executed exporter") + if build.get("sha256") != exporter_sha256: + raise PreflightError("llama trace build SHA-256 does not match the executed exporter") + if manifest.get("revision") != candidate_revision: + raise PreflightError("llama trace build revision does not match the exact candidate revision") + if "test-only manifest harness" in str(build.get("info", "")): + raise PreflightError("llama trace was produced by the test-only manifest harness") + libraries = build.get("runtime_libraries") + if not isinstance(libraries, list) or not libraries: + raise PreflightError("llama trace runtime library identities are missing") + post_libraries = build.get("runtime_libraries_post") + if post_libraries != libraries: + raise PreflightError("llama trace runtime library closure changed during trace generation") + module_monitor = build.get("runtime_module_monitor") + if not isinstance(module_monitor, dict) or set(module_monitor) != { + "mechanism", "checked_after_trace", "project_additions"}: + raise PreflightError("llama trace runtime module monitor is invalid") + if module_monitor["mechanism"] not in {"dyld-add-image", "pre-post-snapshot"}: + raise PreflightError("llama trace runtime module monitor mechanism is invalid") + if module_monitor["checked_after_trace"] is not True: + raise PreflightError("llama trace runtime module monitor did not complete") + if module_monitor["project_additions"] != []: + raise PreflightError("llama trace records a runtime module addition during trace generation") + profile = build.get("runtime_profile") + if not isinstance(profile, dict) or set(profile) != { + "name", "components", "selected_backend_component"}: + raise PreflightError("llama trace runtime profile is invalid") + if profile.get("name") != "sibling-lib": + raise PreflightError("llama trace runtime profile is not the Linux sibling-lib profile") + components = profile.get("components") + if not isinstance(components, list) or components != sorted(components) or ( + len(components) != len(set(components))): + raise PreflightError("llama trace runtime profile components are invalid") + if not {"llama-common", "llama", "ggml", "ggml-base", "ggml-hip"}.issubset(set(components)): + raise PreflightError("llama trace runtime profile is missing required ROCm components") + selected_backend_component = profile.get("selected_backend_component") + if selected_backend_component != "ggml-hip": + raise PreflightError("llama trace selected backend component is not ggml-hip") + roles = set() + paths = set() + found_components = set() + previous_path = None + library_directory = resolved(exporter.parent.parent / "lib") + for library in libraries: + if not isinstance(library, dict) or set(library) != { + "component", "filename", "path", "sha256", "role", "revision"}: + raise PreflightError("llama trace runtime library identity is invalid") + component = library.get("component") + filename = library.get("filename") + path_value = library.get("path") + digest = library.get("sha256") + role = library.get("role") + revision = library.get("revision") + if component not in components or component in found_components: + raise PreflightError("llama trace runtime library component is invalid") + found_components.add(component) + if not isinstance(filename, str) or Path(filename).name != filename: + raise PreflightError("llama trace runtime library filename is invalid") + expected_role = { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + selected_backend_component: "selected-backend", + }.get(component, f"runtime:{component}") + if role != expected_role or role in roles: + raise PreflightError("llama trace runtime library role is invalid") + roles.add(role) + if component in {"llama-common", "ggml-base"}: + if revision != candidate_revision: + raise PreflightError("llama trace runtime library revision mismatch") + elif revision is not None: + raise PreflightError("llama trace runtime library revision is unexpected") + if not isinstance(path_value, str): + raise PreflightError("llama trace runtime library path is invalid") + path = resolved(Path(path_value)) + if path_value != str(path): + raise PreflightError("llama trace runtime library path is not canonical") + if path in paths or (previous_path is not None and str(path) <= str(previous_path)): + raise PreflightError("llama trace runtime library paths are duplicated or unsorted") + paths.add(path) + previous_path = path + if path.parent != library_directory or path.name != filename: + raise PreflightError("llama trace runtime library path differs from the exact runtime profile") + if not path.is_file() or not isinstance(digest, str) or sha256_file(path) != digest: + raise PreflightError("llama trace runtime library SHA-256 mismatch") + if found_components != set(components): + raise PreflightError("llama trace runtime library set differs from the runtime profile") + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": candidate_revision, + "profile": profile["name"], + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in libraries + ], + key=lambda item: item["component"], + ), + } + receipt_sha256 = sha256_bytes(canonical_json(receipt).encode("ascii")) + if build.get("runtime_receipt_sha256") != receipt_sha256: + raise PreflightError("llama trace runtime receipt SHA-256 mismatch") + if approval.get("runtime_profile") != profile or approval.get("runtime_receipt") != receipt: + raise PreflightError("llama trace runtime receipt differs from external exporter approval") + closure = {"pre": libraries, "post": post_libraries} + return sha256_bytes(canonical_json(closure).encode("ascii")), receipt_sha256 + + +def query_runtime_build_attestation( + exporter: Path, + device: str, + *, + exporter_sha256: str, + candidate_revision: str, + approval: dict[str, object]) -> dict[str, object]: + result, _identity = run_approved_executable( + [str(exporter), "--dsv41-attest-build", device], + path=exporter, + runtime_policy=approval, + expected_path=approval["executable_path"], + expected_sha256=approval["executable_sha256"], + label="candidate exporter", + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise PreflightError(f"candidate exporter build attestation failed: {result.stderr.strip()}") + try: + build = strict_json_loads(result.stdout) + except TraceError as error: + raise PreflightError(f"candidate exporter build attestation is invalid: {error}") from error + if not isinstance(build, dict): + raise PreflightError("candidate exporter build attestation is not an object") + manifest = {"revision": candidate_revision, "build": copy.deepcopy(build)} + libraries = manifest["build"].get("runtime_libraries") + if not isinstance(libraries, list): + raise PreflightError("candidate exporter build attestation has no runtime libraries") + manifest["build"]["runtime_libraries_post"] = copy.deepcopy(libraries) + monitor = manifest["build"].get("runtime_module_monitor") + if not isinstance(monitor, dict): + raise PreflightError("candidate exporter build attestation has no runtime monitor") + monitor["checked_after_trace"] = True + validate_runtime_build( + manifest, + exporter=exporter, + exporter_sha256=exporter_sha256, + candidate_revision=candidate_revision, + approval=approval, + ) + return build + + +def bind_candidate_attestation( + output: Path, + attestation: dict[str, str], + accelerator: dict[str, object], + exporter: Path, + exporter_sha256: str, + approval: dict[str, object]) -> None: + manifest_path = safe_trace_path(output, "manifest.json") + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: + raise PreflightError(f"cannot bind candidate attestation: {error}") from error + if manifest.get("accelerator") != accelerator: + raise PreflightError("llama trace accelerator attestation differs from the preflight query") + bound_attestation = dict(attestation) + libraries_sha256, receipt_sha256 = validate_runtime_build( + manifest, + exporter=exporter, + exporter_sha256=exporter_sha256, + candidate_revision=attestation["revision"], + approval=approval, + ) + bound_attestation["runtime_libraries_sha256"] = libraries_sha256 + bound_attestation["runtime_receipt_sha256"] = receipt_sha256 + manifest["candidate"] = bound_attestation + temp = manifest_path.with_suffix(".tmp") + temp.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +def validate_accelerator_attestation( + record: object, + *, + expected_device: str = "ROCm0") -> dict[str, object]: + if not isinstance(record, dict): + raise PreflightError("accelerator attestation is not an object") + required_keys = { + "format", + "version", + "runtime_kind", + "platform", + "backend", + "backend_device", + "backend_description", + "pci_device_id", + "kfd_node", + "gpu_id", + "gfx_target_version", + "architecture", + "source", + } + if set(record) != required_keys: + raise PreflightError("accelerator attestation fields are invalid") + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "backend": "ROCm", + "backend_device": expected_device, + "architecture": "gfx1151", + "gfx_target_version": 110501, + "source": "linux-kfd-sysfs", + } + for key, value in expected.items(): + if record.get(key) != value: + raise PreflightError(f"accelerator attestation {key} mismatch") + if not isinstance(record.get("backend_description"), str) or not record["backend_description"]: + raise PreflightError("accelerator attestation backend description is missing") + pci_device_id = record.get("pci_device_id") + if not isinstance(pci_device_id, str) or re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", pci_device_id) is None: + raise PreflightError("accelerator attestation PCI identity is invalid") + if not isinstance(record.get("kfd_node"), str) or not record["kfd_node"].isdigit(): + raise PreflightError("accelerator attestation KFD node is invalid") + if type(record.get("gpu_id")) is not int or record["gpu_id"] <= 0: + raise PreflightError("accelerator attestation GPU identity is invalid") + return dict(record) + + +def query_accelerator_attestation( + exporter: Path, + device: str, + approval: dict[str, object] | None = None) -> dict[str, object]: + try: + if approval is None: + result = subprocess.run( + [str(exporter), "--dsv41-attest-device", device], + check=False, + capture_output=True, + text=True, + ) + else: + result, _identity = run_approved_executable( + [str(exporter), "--dsv41-attest-device", device], + path=exporter, + runtime_policy=approval, + expected_path=approval["executable_path"], + expected_sha256=approval["executable_sha256"], + label="candidate exporter", + check=False, + capture_output=True, + text=True, + ) + except OSError as error: + raise PreflightError(f"cannot query selected accelerator: {error}") from error + if result.returncode != 0: + detail = result.stderr.strip() or f"exit {result.returncode}" + raise PreflightError(f"selected accelerator query failed: {detail}") + try: + record = strict_json_loads(result.stdout) + except TraceError as error: + raise PreflightError(f"selected accelerator query returned invalid JSON: {error}") from error + return validate_accelerator_attestation(record, expected_device=device) + + +def build_command(args: argparse.Namespace, exporter: Path, output: Path) -> list[str]: + return [ + str(exporter), + "-m", str(resolved(args.model)), + "-bf", str(resolved(args.prompt)), + "-o", str(output), + "-c", str(args.context), + "-n", str(args.decode_steps), + "-b", str(args.batch), + "-ub", str(args.ubatch), + "--device", args.device, + "-ngl", str(args.gpu_layers), + "-fa", "on", + "-ctk", "f16", + "-ctv", "f16", + "--load-mode", "none", + "--expert-cache-slots", str(args.expert_cache_slots), + "--expert-cache-mib", str(args.expert_cache_mib), + ] + + +def validate_runtime_config(args: argparse.Namespace) -> None: + if args.batch != ADMITTED_BATCH: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require batch {ADMITTED_BATCH}, found {args.batch}") + if args.ubatch != ADMITTED_UBATCH: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require admitted ubatch {ADMITTED_UBATCH}, found {args.ubatch}") + if args.expert_cache_slots != REQUIRED_EXPERT_SLOTS: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require {REQUIRED_EXPERT_SLOTS} expert cache slots, " + f"found {args.expert_cache_slots}") + if args.expert_cache_mib != REQUIRED_EXPERT_CACHE_MIB: + raise PreflightError( + f"DeepSeek V4.1 correctness runs require {REQUIRED_EXPERT_CACHE_BYTES} expert cache bytes " + f"({REQUIRED_EXPERT_CACHE_MIB} MiB), found {args.expert_cache_mib} MiB") + if args.device != "ROCm0": + raise PreflightError(f"DeepSeek V4.1 correctness runs require device ROCm0, found {args.device}") + if args.gpu_layers != 99: + raise PreflightError(f"DeepSeek V4.1 correctness runs require 99 GPU layers, found {args.gpu_layers}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed launcher for llama.cpp DeepSeek V4.1 traces") + parser.add_argument("--exporter", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--candidate-revision", required=True) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--candidate-diff-sha256", required=True) + parser.add_argument("--candidate-exporter-policy-id", required=True) + parser.add_argument("--prompt-builder-policy-id", required=True) + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) + parser.add_argument("--corpus-name", choices=sorted(CORPUS_SHA256), required=True) + parser.add_argument("--corpus-sha256", required=True) + parser.add_argument("--prompt-provenance", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--prompt", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) + parser.add_argument("--context", type=int, default=32768) + parser.add_argument("--decode-steps", type=int, default=8) + parser.add_argument("--batch", type=int, default=ADMITTED_BATCH) + parser.add_argument("--ubatch", type=int, default=ADMITTED_UBATCH) + parser.add_argument("--device", default="ROCm0") + parser.add_argument("--expert-cache-slots", type=int, default=REQUIRED_EXPERT_SLOTS) + parser.add_argument("--expert-cache-mib", type=int, default=REQUIRED_EXPERT_CACHE_MIB) + parser.add_argument("--gpu-layers", type=int, default=99) + parser.add_argument("--signer-principal", required=True) + parser.add_argument("--signing-key", type=Path, required=True) + parser.add_argument("--execution-challenge", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--authorization-issued-unix", type=int, required=True) + parser.add_argument("--authorization-expires-unix", type=int, required=True) + parser.add_argument("--preflight-only", action="store_true") + args = parser.parse_args() + + model_descriptor = -1 + watchdog_descriptor = -1 + try: + validate_runtime_config(args) + reject_loader_overrides() + output = resolved(args.output) + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(output,), + ) + candidate_policy, candidate_policy_sha256 = candidate_exporter_approval( + args.candidate_exporter_policy_id, + policies=approval_policy.candidate_exporters, + ) + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + args.prompt_builder_policy_id, + policies=approval_policy.prompt_builders, + ) + if args.corpus_sha256 != CORPUS_SHA256[args.corpus_name]: + raise PreflightError(f"corpus SHA-256 mismatch for {args.corpus_name}") + validate_signing_identity( + args.signing_key, + args.signer_principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + forbidden_root=output, + ) + exporter = args.exporter + exporter_identity = approved_executable_identity( + exporter, + install_root=candidate_policy["install_root"], + expected_owner_uid=candidate_policy["install_owner_uid"], + expected_path=candidate_policy["executable_path"], + expected_sha256=candidate_policy["executable_sha256"], + label="candidate exporter", + ) + runtime_identities = approved_runtime_file_identities( + candidate_policy, label="candidate exporter") + helper_identity = approved_containment_helper_identity( + candidate_policy, label="candidate exporter") + candidate_trust = install_trust_evidence( + exporter_identity, runtime_identities, (helper_identity,)) + exporter_sha256 = exporter_identity.sha256 + if args.candidate_revision != candidate_policy["revision"] or ( + args.base_revision != candidate_policy["base_revision"]) or ( + args.candidate_diff_sha256 != candidate_policy["diff_sha256"]): + raise PreflightError("candidate arguments differ from external exporter approval") + repo = resolved(args.repo) + if git_output(repo, "rev-parse", "HEAD").decode("ascii").strip() != ( + approval_policy.verifier_revision): + raise PreflightError("candidate verifier checkout differs from the external approval policy") + if repo != approved_source_root(prompt_policy) or candidate_policy["revision"] != prompt_policy["revision"]: + raise PreflightError("candidate repository or revision differs from prompt builder approval") + model_descriptor, model_identity = open_model_descriptor(args.model) + model_sha256 = str(model_identity["sha256"]) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + provenance = validate_prompt_provenance( + args.prompt_provenance, + prompt=args.prompt, + corpus_name=args.corpus_name, + corpus_sha256=args.corpus_sha256, + model_sha256=model_sha256, + target_tokens=args.context - args.decode_steps, + context=args.context, + decode_steps=args.decode_steps, + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, + ) + prompt_trust_sha256 = provenance["record"]["builder_install_trust_sha256"] + authorization = execution_authorization( + lane=CANDIDATE_LANE, + challenge=args.execution_challenge, + run_id=args.run_id, + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), + approvals={ + "candidate_exporter": approval_binding( + "candidate_exporter", + args.candidate_exporter_policy_id, + candidate_policy_sha256, + install_trust_sha256(candidate_trust), + ), + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + prompt_trust_sha256, + ), + }, + ) + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + pre_runtime_build = query_runtime_build_attestation( + exporter, + args.device, + exporter_sha256=exporter_sha256, + candidate_revision=args.candidate_revision, + approval=candidate_policy, + ) + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + accelerator = query_accelerator_attestation(exporter, args.device, candidate_policy) + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + if args.preflight_only: + audit = run_strix_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + repo=args.repo, + busy_patterns=args.busy_pattern, + ) + audit["accelerator"] = accelerator + print(json.dumps(audit, sort_keys=True, separators=(",", ":"))) + return 0 + + attestation = candidate_attestation( + args, + exporter, + exporter_sha256, + args.candidate_exporter_policy_id, + candidate_policy_sha256, + candidate_policy, + approval_policy.verifier_revision, + candidate_trust, + ) + if output.exists() and any(output.iterdir()): + raise PreflightError(f"trace output directory is not empty: {output}") + preflight_audit = run_strix_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + repo=args.repo, + busy_patterns=args.busy_pattern, + ) + preflight_audit["runtime"] = "llama.cpp" + preflight_audit["accelerator"] = accelerator + preflight_audit["config"] = { + "context": args.context, + "decode_steps": args.decode_steps, + "batch": args.batch, + "ubatch": args.ubatch, + "device": args.device, + "device_architecture": accelerator["architecture"], + "device_pci_id": accelerator["pci_device_id"], + "expert_cache_slots": args.expert_cache_slots, + "expert_cache_mib": args.expert_cache_mib, + "gpu_layers": args.gpu_layers, + } + watchdog_descriptor, watchdog_authority = open_watchdog_namespace_authority( + preflight_audit["watchdog"]) + preflight_audit["watchdog"]["namespace_authority"] = watchdog_authority + pre_audits = write_audits(Path(str(output) + ".audit") / "pre", preflight_audit) + pre_audit_digests = seal_audits(pre_audits) + environment = os.environ.copy() + environment["DSV41_TRACE_MEMORY_AUDIT"] = pre_audits["memory"] + environment["DSV41_TRACE_SWAP_AUDIT"] = pre_audits["swap"] + environment["DSV41_TRACE_WATCHDOG_AUDIT"] = pre_audits["watchdog"] + environment["DSV41_TOKENIZER_POLICY"] = canonical_json(prompt_policy["tokenizer"]) + environment["DSV41_MODEL_DESCRIPTOR"] = str(model_descriptor) + environment["DSV41_MODEL_DESCRIPTOR_IDENTITY"] = canonical_json(model_identity) + environment["DSV41_WATCHDOG_PIDFD"] = str(watchdog_descriptor) + command = build_command(args, exporter, output) + print("exec:", shlex.join(command), file=sys.stderr) + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + result, executed_identity = run_approved_executable( + command, + path=exporter, + runtime_policy=candidate_policy, + expected_path=candidate_policy["executable_path"], + expected_sha256=candidate_policy["executable_sha256"], + label="candidate exporter", + env=environment, + check=False, + retained_fds=(model_descriptor, watchdog_descriptor), + ) + verify_model_descriptor(model_descriptor, model_identity) + verify_watchdog_namespace_authority(watchdog_descriptor, watchdog_authority) + if executed_identity != exporter_identity: + raise PreflightError("candidate exporter execution identity differs from external approval") + if result.returncode != 0: + return result.returncode + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + post_runtime_build = query_runtime_build_attestation( + exporter, + args.device, + exporter_sha256=exporter_sha256, + candidate_revision=args.candidate_revision, + approval=candidate_policy, + ) + if post_runtime_build != pre_runtime_build: + raise PreflightError("candidate exporter build identity changed during trace execution") + verify_approved_executable_identity(exporter, exporter_identity, label="candidate exporter") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + verify_sealed_audits(pre_audits, pre_audit_digests) + postflight_audit = run_strix_preflight( + model=args.model, + prompt=args.prompt, + output=args.output, + repo=args.repo, + busy_patterns=args.busy_pattern, + ) + verify_watchdog_namespace_authority(watchdog_descriptor, watchdog_authority) + postflight_audit["watchdog"]["namespace_authority"] = watchdog_authority + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + post_accelerator = query_accelerator_attestation( + exporter, args.device, candidate_policy) + if post_accelerator != accelerator: + raise PreflightError("selected accelerator identity changed during trace execution") + verify_approved_runtime_file_identities( + runtime_identities, label="candidate exporter") + postflight_audit["runtime"] = "llama.cpp" + postflight_audit["accelerator"] = post_accelerator + post_audits = write_audits(Path(str(output) + ".audit") / "post", postflight_audit) + bind_embedded_audits(output, {"pre": pre_audits, "post": post_audits}) + bind_prompt_provenance(output, provenance) + bind_candidate_attestation( + output, attestation, accelerator, exporter, exporter_sha256, candidate_policy) + bind_execution_authorization(output, authorization) + unsealed_manifest = strict_json_loads( + safe_trace_path(output, "manifest.json").read_text(encoding="ascii")) + if not isinstance(unsealed_manifest, dict) or validate_tokenizer_policy( + unsealed_manifest.get("config", {}).get("tokenizer")) != prompt_policy["tokenizer"]: + raise PreflightError("candidate manifest tokenizer policy differs from external approval") + seal_bundle( + output, + private_key=args.signing_key, + principal=args.signer_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + candidate_exporter_policies=approval_policy.candidate_exporters, + ds4_exporter_policies={}, + prompt_builder_policies=approval_policy.prompt_builders, + expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + expected_approval_policy_sha256=approval_policy.sha256, + expected_verifier_revision=approval_policy.verifier_revision, + trusted_signers=APPROVED_TRACE_SIGNERS, + ) + bundle = TraceBundle( + output, + verifier=TraceVerifier.production( + args.signer_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + ), + ) + if bundle.manifest.get("runtime") != "llama.cpp": + raise PreflightError("llama exporter wrote a non-llama.cpp trace") + if bundle.manifest.get("build", {}).get("sha256") != exporter_sha256: + raise PreflightError("llama trace build SHA-256 does not match the executed exporter") + if bundle.manifest.get("model", {}).get("sha256") != MODEL_SHA256: + raise PreflightError("llama trace model SHA-256 does not match the published GGUF") + return 0 + except (PreflightError, TraceError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + finally: + for descriptor in (watchdog_descriptor, model_descriptor): + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + print(f"error: cannot close retained descriptor {descriptor}: {error}", file=sys.stderr) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/run_matrix.py b/tools/deepseek-v41-trace/run_matrix.py new file mode 100644 index 000000000000..6b860c8f1f35 --- /dev/null +++ b/tools/deepseek-v41-trace/run_matrix.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from preflight import PreflightError, require_nvme_path, resolved, run_strix_preflight +from trace_format import ( + ADMITTED_BATCH, + ADMITTED_UBATCH, + APPROVED_CANDIDATE_EXPORTERS, + APPROVED_PROMPT_BUILDERS, + APPROVED_TRACE_SIGNERS, + CANDIDATE_LANE, + CORPUS_SHA256, + MODEL_SHA256, + REQUIRED_EXPERT_CACHE_MIB, + REQUIRED_EXPERT_SLOTS, + TraceError, + approval_binding, + approved_containment_helper_identity, + approved_executable_identity, + approved_prompt_record, + approved_runtime_file_identities, + candidate_exporter_approval, + execution_authorization, + install_trust_evidence, + install_trust_sha256, + load_executable_approval_policy, + prompt_builder_approval, + reject_loader_overrides, + runtime_build_evidence_sha256, + run_approved_executable, + sha256_file, + strict_json_loads, + tokenizer_policy_sha256, + validate_signing_identity, + validate_runtime_build_evidence, + validate_tokenizer_policy, + verify_approved_executable_identity, + verify_approved_runtime_file_identities, +) + +CORPORA = ( + "correctness-prose.txt", + "correctness-code.txt", + "correctness-structured.txt", + "correctness-numeric.txt", +) + + +def approved_source_root(builder_policy: dict[str, object]) -> Path: + try: + return Path(str(builder_policy["source_root"])).expanduser().resolve(strict=True) + except (KeyError, OSError) as error: + raise PreflightError(f"prompt builder approved source root is invalid: {error}") from error + + +def query_prompt_builder_runtime_build( + builder: Path, + builder_policy: dict[str, object]) -> dict[str, object]: + result, _identity = run_approved_executable( + [str(builder), "--dsv41-attest-build"], + path=builder, + runtime_policy=builder_policy, + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"prompt builder build attestation failed: {result.stderr.strip()}") + try: + record = strict_json_loads(result.stdout) + return validate_runtime_build_evidence(record, builder_policy, label="prompt builder") + except TraceError as error: + raise RuntimeError(f"prompt builder build attestation is invalid: {error}") from error + + +def run(command: list[str]) -> None: + displayed = list(command) + if "--signing-key" in displayed: + index = displayed.index("--signing-key") + if index + 1 < len(displayed): + displayed[index + 1] = "" + print("exec:", " ".join(displayed), file=sys.stderr) + result = subprocess.run(command, check=False) + if result.returncode != 0: + raise RuntimeError(f"command failed with status {result.returncode}") + + +def file_identity(path: Path) -> tuple[int, int, int, int, int]: + record = path.stat() + return ( + record.st_dev, + record.st_ino, + record.st_size, + record.st_mtime_ns, + record.st_ctime_ns, + ) + + +def prepare_prompt( + *, + builder: Path, + builder_approval_id: str, + builder_policy: dict[str, object], + builder_policy_sha256: str, + model: Path, + corpus: Path, + source_corpus: Path, + corpus_name: str, + corpus_sha256: str, + output: Path, + context: int, + decode_steps: int, +) -> dict[str, object]: + target_tokens = context - decode_steps + expected_prompt = approved_prompt_record( + builder_policy, + corpus_name=corpus_name, + context=context, + decode_steps=decode_steps, + ) + builder_identity = approved_executable_identity( + builder, + install_root=builder_policy["install_root"], + expected_owner_uid=builder_policy["install_owner_uid"], + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + ) + runtime_identities = approved_runtime_file_identities( + builder_policy, label="prompt builder") + helper_identity = approved_containment_helper_identity( + builder_policy, label="prompt builder") + source_root_lexical = Path(builder_policy["source_root"]) + source_root_resolved = approved_source_root(builder_policy) + source_corpus_lexical = source_corpus + source_corpus_resolved = source_corpus_lexical.resolve(strict=True) + expected_source = ( + source_root_resolved / "tests" / "corpus" / corpus_name).resolve(strict=True) + if source_corpus_resolved != expected_source or sha256_file(source_corpus_resolved) != corpus_sha256 or ( + sha256_file(corpus) != corpus_sha256): + raise RuntimeError("prompt builder corpus path or bytes differ from external approval") + source_identity = file_identity(source_corpus_resolved) + corpus_identity = file_identity(corpus) + verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") + tokenizer = validate_tokenizer_policy(builder_policy["tokenizer"]) + pre_runtime_build = query_prompt_builder_runtime_build(builder, builder_policy) + command = [ + str(builder), + "--model", str(model), + "--corpus", str(corpus), + "--output", str(output), + "--tokens", str(target_tokens), + "--tokenizer-add-bos", str(tokenizer["add_bos"]).lower(), + "--tokenizer-parse-special", str(tokenizer["parse_special"]).lower(), + "--tokenizer-detokenize-special", str(tokenizer["detokenize_special"]).lower(), + "--tokenizer-remove-leading-bos", str( + tokenizer["remove_leading_bos_before_detokenize"]).lower(), + "--tokenizer-require-round-trip", str(tokenizer["require_round_trip"]).lower(), + ] + print("exec:", " ".join(command), file=sys.stderr) + result, executed_identity = run_approved_executable( + command, + path=builder, + runtime_policy=builder_policy, + expected_path=builder_policy["executable_path"], + expected_sha256=builder_policy["executable_sha256"], + label="prompt builder", + check=False, + capture_output=True, + text=True, + ) + if executed_identity != builder_identity: + raise RuntimeError("prompt builder execution identity differs from external approval") + verify_approved_executable_identity(builder, builder_identity, label="prompt builder") + verify_approved_runtime_file_identities(runtime_identities, label="prompt builder") + if file_identity(source_corpus_resolved) != source_identity or file_identity(corpus) != corpus_identity or ( + sha256_file(source_corpus_resolved) != corpus_sha256) or sha256_file(corpus) != corpus_sha256: + raise RuntimeError("prompt builder corpus changed during execution") + if result.returncode != 0: + raise RuntimeError(f"prompt builder failed: {result.stderr.strip()}") + try: + native_record = strict_json_loads(result.stdout) + except TraceError as error: + raise RuntimeError(f"prompt builder returned invalid JSON: {error}") from error + if not isinstance(native_record, dict) or set(native_record) != { + "target_tokens", "actual_tokens", "byte_count", "tokenizer", + "runtime_build", "temporary_directory"}: + raise RuntimeError("prompt builder returned an invalid result schema") + if native_record.get("target_tokens") != target_tokens or native_record.get("actual_tokens") != target_tokens: + raise RuntimeError("prompt builder did not produce the requested token count") + if type(native_record.get("byte_count")) is not int or native_record["byte_count"] != output.stat().st_size: + raise RuntimeError("prompt builder byte count does not match its output") + if validate_tokenizer_policy(native_record.get("tokenizer")) != tokenizer: + raise RuntimeError("prompt builder tokenizer policy differs from external approval") + runtime_build = validate_runtime_build_evidence( + native_record.get("runtime_build"), builder_policy, label="prompt builder") + if runtime_build != pre_runtime_build: + raise RuntimeError("prompt builder runtime build changed during prompt construction") + temporary_directory = native_record["temporary_directory"] + expected_temporary_directory = os.environ.get("TMPDIR") + if not expected_temporary_directory or temporary_directory != str(resolved(Path(expected_temporary_directory))): + raise RuntimeError("prompt builder did not attest the selected temporary directory") + prompt_sha256 = sha256_file(output) + prompt_byte_count = output.stat().st_size + if prompt_sha256 != expected_prompt["prompt_sha256"] or ( + prompt_byte_count != expected_prompt["prompt_byte_count"]): + raise RuntimeError("prompt builder output differs from external approval") + trust_evidence = install_trust_evidence( + builder_identity, runtime_identities, (helper_identity,)) + record = { + "format": "dsv41-prompt-provenance", + "version": 2, + "corpus_name": corpus_name, + "corpus_sha256": corpus_sha256, + "corpus_path": str(source_corpus_resolved), + "corpus_lexical_path": str(source_corpus_lexical), + "corpus_resolved_path": str(source_corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), + "model_sha256": MODEL_SHA256, + "prompt_sha256": prompt_sha256, + "prompt_byte_count": prompt_byte_count, + "context": context, + "decode_steps": decode_steps, + "builder_approval_id": builder_approval_id, + "builder_approval_sha256": builder_policy_sha256, + "builder_path": str(builder), + "builder_sha256": builder_identity.sha256, + "builder_revision": builder_policy["revision"], + "builder_runtime_profile": builder_policy["runtime_profile"], + "tokenizer": tokenizer, + "builder_runtime_build": runtime_build, + "builder_runtime_build_sha256": runtime_build_evidence_sha256( + runtime_build, builder_policy, label="prompt builder"), + "builder_install_trust": trust_evidence, + "builder_install_trust_sha256": install_trust_sha256(trust_evidence), + "target_tokens": target_tokens, + "actual_tokens": target_tokens, + } + provenance_path = output.with_suffix(output.suffix + ".provenance.json") + provenance_path.write_text( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + record.update({ + "path": str(output), + "provenance_path": str(provenance_path), + "provenance_sha256": sha256_file(provenance_path), + }) + return record + + +def main() -> int: + parser = argparse.ArgumentParser(description="Capture the DeepSeek V4.1 llama.cpp corpus matrix") + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--llama-runner", type=Path, required=True) + parser.add_argument("--llama-exporter", type=Path, required=True) + parser.add_argument("--llama-prompt-builder", type=Path, required=True) + parser.add_argument("--candidate-revision", required=True) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--candidate-diff-sha256", required=True) + parser.add_argument("--candidate-exporter-policy-id", required=True) + parser.add_argument("--prompt-builder-policy-id", required=True) + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) + parser.add_argument("--llama-only", action="store_true") + parser.add_argument("--contexts", type=int, nargs="+", default=[32768]) + parser.add_argument("--ubatches", type=int, nargs="+", default=[ADMITTED_UBATCH]) + parser.add_argument("--decode-steps", type=int, default=8) + parser.add_argument("--batch", type=int, default=ADMITTED_BATCH) + parser.add_argument("--device", default="ROCm0") + parser.add_argument("--expert-cache-slots", type=int, default=REQUIRED_EXPERT_SLOTS) + parser.add_argument("--expert-cache-mib", type=int, default=REQUIRED_EXPERT_CACHE_MIB) + parser.add_argument("--busy-pattern", action="append", default=["ds4-v41", "DeepSeek-V4.1"]) + parser.add_argument("--signer-principal", required=True) + parser.add_argument("--signing-key", type=Path, required=True) + parser.add_argument("--execution-challenge", required=True) + parser.add_argument("--run-id-prefix", required=True) + parser.add_argument("--authorization-issued-unix", type=int, required=True) + parser.add_argument("--authorization-expires-unix", type=int, required=True) + args = parser.parse_args() + + try: + if args.ubatches != [ADMITTED_UBATCH]: + raise PreflightError( + f"DeepSeek V4.1 correctness matrix requires admitted ubatch [{ADMITTED_UBATCH}]") + if args.batch != ADMITTED_BATCH: + raise PreflightError(f"DeepSeek V4.1 correctness matrix requires batch {ADMITTED_BATCH}") + if args.device != "ROCm0": + raise PreflightError("DeepSeek V4.1 correctness matrix requires device ROCm0") + if args.expert_cache_slots != REQUIRED_EXPERT_SLOTS: + raise PreflightError( + f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_SLOTS} expert cache slots") + if args.expert_cache_mib != REQUIRED_EXPERT_CACHE_MIB: + raise PreflightError( + f"DeepSeek V4.1 correctness matrix requires {REQUIRED_EXPERT_CACHE_MIB} MiB expert cache") + reject_loader_overrides() + output_candidate = resolved(args.output) + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(output_candidate,), + ) + candidate_policy, candidate_policy_sha256 = candidate_exporter_approval( + args.candidate_exporter_policy_id, + policies=approval_policy.candidate_exporters, + ) + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + args.prompt_builder_policy_id, + policies=approval_policy.prompt_builders, + ) + candidate_identity = approved_executable_identity( + args.llama_exporter, + install_root=candidate_policy["install_root"], + expected_owner_uid=candidate_policy["install_owner_uid"], + expected_path=candidate_policy["executable_path"], + expected_sha256=candidate_policy["executable_sha256"], + label="candidate exporter", + ) + candidate_runtime_identities = approved_runtime_file_identities( + candidate_policy, label="candidate exporter") + candidate_helper_identity = approved_containment_helper_identity( + candidate_policy, label="candidate exporter") + candidate_trust = install_trust_evidence( + candidate_identity, candidate_runtime_identities, (candidate_helper_identity,)) + prompt_builder = args.llama_prompt_builder + prompt_identity = approved_executable_identity( + prompt_builder, + install_root=prompt_policy["install_root"], + expected_owner_uid=prompt_policy["install_owner_uid"], + expected_path=prompt_policy["executable_path"], + expected_sha256=prompt_policy["executable_sha256"], + label="prompt builder", + ) + prompt_runtime_identities = approved_runtime_file_identities( + prompt_policy, label="prompt builder") + prompt_helper_identity = approved_containment_helper_identity( + prompt_policy, label="prompt builder") + prompt_trust = install_trust_evidence( + prompt_identity, prompt_runtime_identities, (prompt_helper_identity,)) + if args.candidate_revision != candidate_policy["revision"] or ( + args.base_revision != candidate_policy["base_revision"]) or ( + args.candidate_diff_sha256 != candidate_policy["diff_sha256"]): + raise PreflightError("matrix candidate identity differs from external exporter approval") + execution_authorization( + lane=CANDIDATE_LANE, + challenge=args.execution_challenge, + run_id=f"{args.run_id_prefix}-preflight", + issued_unix=args.authorization_issued_unix, + expires_unix=args.authorization_expires_unix, + approval_policy_sha256=approval_policy.sha256, + verifier_revision=approval_policy.verifier_revision, + tokenizer_policy_sha256_value=tokenizer_policy_sha256(prompt_policy["tokenizer"]), + approvals={ + "candidate_exporter": approval_binding( + "candidate_exporter", + args.candidate_exporter_policy_id, + candidate_policy_sha256, + install_trust_sha256(candidate_trust), + ), + "prompt_builder": approval_binding( + "prompt_builder", + args.prompt_builder_policy_id, + prompt_policy_sha256, + install_trust_sha256(prompt_trust), + ), + }, + ) + if not args.llama_only: + raise PreflightError( + "cross-runtime capture must run on separate Strix and Apple hosts; " + "use --llama-only here and compare completed bundles with trace_format.py") + validate_signing_identity( + args.signing_key, + args.signer_principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + forbidden_root=output_candidate, + ) + repo = resolved(args.repo) + revision = subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + stderr=subprocess.STDOUT, + ).decode("ascii").strip() + if revision != approval_policy.verifier_revision: + raise PreflightError("matrix verifier checkout differs from the external approval policy") + output = require_nvme_path(output_candidate, "matrix output") + model = require_nvme_path(args.model, "model") + if not model.is_file(): + raise PreflightError(f"model is not a file: {model}") + model_sha256 = sha256_file(model) + if model_sha256 != MODEL_SHA256: + raise PreflightError(f"published model SHA-256 mismatch: expected {MODEL_SHA256}, found {model_sha256}") + initial_corpus = require_nvme_path( + repo / "tests" / "corpus" / CORPORA[0], + "repository corpus", + ) + run_strix_preflight( + model=model, + prompt=initial_corpus, + output=output, + repo=repo, + busy_patterns=args.busy_pattern, + ) + if repo != approved_source_root(prompt_policy) or args.candidate_revision != prompt_policy["revision"]: + raise PreflightError("matrix repository or revision differs from prompt builder approval") + if output.exists() and any(output.iterdir()): + raise PreflightError(f"matrix output directory is not empty: {output}") + inputs = output / "inputs" + sources = inputs / "sources" + prompts = inputs / "prompts" + sources.mkdir(parents=True, exist_ok=True) + prompts.mkdir(parents=True, exist_ok=True) + corpus_records = [] + for name in CORPORA: + source = require_nvme_path(repo / "tests" / "corpus" / name, "repository corpus") + if not source.is_file(): + raise PreflightError(f"repository corpus is missing: {source}") + destination = sources / name + shutil.copyfile(source, destination) + source_sha256 = sha256_file(destination) + if source_sha256 != CORPUS_SHA256[name]: + raise PreflightError( + f"repository corpus SHA-256 mismatch for {name}: expected {CORPUS_SHA256[name]}, found {source_sha256}") + corpus_records.append({ + "name": name, + "source": str(source), + "path": str(destination), + "byte_count": destination.stat().st_size, + "sha256": source_sha256, + }) + + results = [] + prompt_records = [] + for context in args.contexts: + if context < 32768 or context > 131072: + raise PreflightError(f"context is outside the supported 32768..131072 matrix: {context}") + target_tokens = context - args.decode_steps + if target_tokens < 1: + raise PreflightError("decode steps leave no room for prompt tokens") + prepared_prompts = {} + for corpus in corpus_records: + stem = Path(corpus["name"]).stem + prompt = prompts / f"{stem}-c{context}.txt" + run_strix_preflight( + model=model, + prompt=Path(corpus["path"]), + output=prompt, + repo=repo, + busy_patterns=args.busy_pattern, + ) + prepared = prepare_prompt( + builder=resolved(args.llama_prompt_builder), + builder_approval_id=args.prompt_builder_policy_id, + builder_policy=prompt_policy, + builder_policy_sha256=prompt_policy_sha256, + model=model, + corpus=Path(corpus["path"]), + source_corpus=Path(corpus["source"]), + corpus_name=corpus["name"], + corpus_sha256=corpus["sha256"], + output=prompt, + context=context, + decode_steps=args.decode_steps, + ) + prepared.update({"corpus": corpus["name"], "context": context}) + prepared_prompts[corpus["name"]] = prepared + prompt_records.append(prepared) + for ubatch in args.ubatches: + for corpus in corpus_records: + stem = Path(corpus["name"]).stem + case = f"{stem}-c{context}-ub{ubatch}" + run_id = f"{args.run_id_prefix}-{case}" + llama_output = output / "llama" / case + prompt = prepared_prompts[corpus["name"]]["path"] + provenance = prepared_prompts[corpus["name"]]["provenance_path"] + common = [ + "--model", str(model), + "--prompt", prompt, + "--prompt-provenance", provenance, + "--corpus-name", corpus["name"], + "--corpus-sha256", corpus["sha256"], + "--context", str(context), + "--decode-steps", str(args.decode_steps), + ] + for pattern in args.busy_pattern: + common.extend(["--busy-pattern", pattern]) + run([ + sys.executable, + str(resolved(args.llama_runner)), + "--exporter", str(resolved(args.llama_exporter)), + "--repo", str(repo), + "--candidate-revision", args.candidate_revision, + "--base-revision", args.base_revision, + "--candidate-diff-sha256", args.candidate_diff_sha256, + "--candidate-exporter-policy-id", args.candidate_exporter_policy_id, + "--prompt-builder-policy-id", args.prompt_builder_policy_id, + "--approval-policy", str(resolved(args.approval_policy)), + "--approval-signature", str(resolved(args.approval_signature)), + "--approval-principal", args.approval_principal, + "--output", str(llama_output), + "--batch", str(args.batch), + "--ubatch", str(ubatch), + "--device", args.device, + "--expert-cache-slots", str(args.expert_cache_slots), + "--expert-cache-mib", str(args.expert_cache_mib), + "--signer-principal", args.signer_principal, + "--signing-key", str(resolved(args.signing_key)), + "--execution-challenge", args.execution_challenge, + "--run-id", run_id, + "--authorization-issued-unix", str(args.authorization_issued_unix), + "--authorization-expires-unix", str(args.authorization_expires_unix), + *common, + ]) + results.append({ + "case": case, + "status": "BRINGUP TRACE CAPTURED", + "cross_runtime_status": "INCOMPLETE", + "trace": str(llama_output), + "run_id": run_id, + }) + + summary = { + "status": "BRINGUP TRACE CAPTURED", + "mode": "llama-only", + "cross_runtime_status": "INCOMPLETE", + "model": str(model), + "model_sha256": model_sha256, + "candidate_revision": args.candidate_revision, + "base_revision": args.base_revision, + "candidate_diff_sha256": args.candidate_diff_sha256, + "signer_principal": args.signer_principal, + "execution_challenge": args.execution_challenge, + "run_id_prefix": args.run_id_prefix, + "corpora": corpus_records, + "prompts": prompt_records, + "contexts": args.contexts, + "ubatches": args.ubatches, + "decode_steps": args.decode_steps, + "target_prompt_tokens": { + str(context): context - args.decode_steps for context in args.contexts + }, + "cases": results, + } + (output / "summary.json").write_text( + json.dumps(summary, sort_keys=True, separators=(",", ":")) + "\n", + encoding="ascii", + ) + return 0 + except (PreflightError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/test-host-attestation.cpp b/tools/deepseek-v41-trace/test-host-attestation.cpp new file mode 100644 index 000000000000..4676df557ddc --- /dev/null +++ b/tools/deepseek-v41-trace/test-host-attestation.cpp @@ -0,0 +1,223 @@ +#include "host-attestation.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void write_file(const fs::path & path, const std::string & value) { + fs::create_directories(path.parent_path()); + std::ofstream output(path); + output << value; + if (!output) { + throw std::runtime_error("cannot write test fixture"); + } +} + +template +static void require_failure(Function function, const std::string & expected) { + try { + function(); + } catch (const std::runtime_error & error) { + if (std::string(error.what()).find(expected) == std::string::npos) { + throw; + } + return; + } + throw std::runtime_error("expected failure containing: " + expected); +} + +int main() { + const fs::path root = fs::canonical(fs::temp_directory_path()) / + ("dsv41-host-attestation-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + try { + const fs::path xfs = root / "mnt" / "models"; + const fs::path btrfs = root / "home"; + const fs::path rotating = root / "rotating"; + const fs::path ram = root / "ram"; + const fs::path network = root / "network"; + const fs::path missing = root / "missing"; + const fs::path forbidden = root / "forbidden"; + fs::create_directories(xfs); + fs::create_directories(btrfs); + fs::create_directories(rotating); + fs::create_directories(ram); + fs::create_directories(network); + fs::create_directories(missing); + fs::create_directories(forbidden); + write_file(xfs / "model.gguf", "model"); + write_file(btrfs / "corpus.txt", "corpus"); + write_file(rotating / "data", "data"); + write_file(ram / "data", "data"); + write_file(network / "data", "data"); + write_file(missing / "data", "data"); + fs::create_directory_symlink(ram, xfs / "escape"); + fs::create_directory_symlink(xfs, forbidden / "escape"); + fs::create_directory_symlink(btrfs, root / "tmp-link"); + dsv41::require_usable_directory(btrfs, "TMPDIR"); + require_failure( + [&]() { + dsv41::require_usable_directory(root / "tmp-link", "TMPDIR"); + }, + "must not be a symlink"); + require_failure( + [&]() { + dsv41::require_usable_directory(fs::path((root / "tmp-link").string() + "/"), "TMPDIR"); + }, + "must not be a symlink"); + require_failure( + [&]() { + dsv41::require_usable_directory(root / "tmp-link" / ".", "TMPDIR"); + }, + "must not be a symlink"); + fs::create_directories(btrfs / "child"); + require_failure( + [&]() { + dsv41::require_usable_directory(root / "tmp-link" / "child", "TMPDIR"); + }, + "must not be a symlink"); + + const fs::path sys = root / "sys"; + const fs::path nvme1 = sys / "devices" / "pci" / "block" / "nvme1n1"; + const fs::path nvme0 = sys / "devices" / "pci" / "block" / "nvme0n1"; + const fs::path nvme0p3 = nvme0 / "nvme0n1p3"; + const fs::path sdb = sys / "devices" / "pci" / "block" / "sdb"; + const fs::path sdb1 = sdb / "sdb1"; + write_file(nvme1 / "queue" / "rotational", "0\n"); + write_file(nvme0 / "queue" / "rotational", "0\n"); + write_file(sdb / "queue" / "rotational", "1\n"); + fs::create_directories(nvme0p3); + fs::create_directories(sdb1); + fs::create_directories(sys / "dev" / "block"); + fs::create_directory_symlink(nvme1, sys / "dev" / "block" / "259:0"); + fs::create_directory_symlink(nvme0p3, sys / "dev" / "block" / "259:3"); + fs::create_directory_symlink(sdb1, sys / "dev" / "block" / "8:17"); + write_file(sys / "class" / "block" / "nvme0n1p3" / "dev", "259:3\n"); + + const fs::path mountinfo = root / "mountinfo"; + write_file( + mountinfo, + "1 0 259:0 / " + fs::canonical(xfs).string() + " rw - xfs /dev/nvme1n1 rw\n" + + "2 0 0:35 /home " + fs::canonical(btrfs).string() + " rw - btrfs /dev/nvme0n1p3[/home] rw\n" + + "3 0 8:17 / " + fs::canonical(rotating).string() + " rw - ext4 /dev/sdb1 rw\n" + + "4 0 0:42 / " + fs::canonical(ram).string() + " rw - tmpfs tmpfs rw\n" + + "5 0 0:43 / " + fs::canonical(network).string() + " rw - nfs server:/share rw\n" + + "6 0 240:1 / " + fs::canonical(missing).string() + " rw - ext4 /dev/missing rw\n"); + + const dsv41::storage_attestation xfs_attestation = + dsv41::require_nvme_path(xfs / "model.gguf", "model", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + if (xfs_attestation.filesystem_type != "xfs" || xfs_attestation.nvme_device != "nvme1n1") { + throw std::runtime_error("xfs NVMe attestation mismatch"); + } + const dsv41::storage_attestation btrfs_attestation = + dsv41::require_nvme_path(btrfs / "new" / "trace", "trace", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + if (btrfs_attestation.filesystem_type != "btrfs" || + btrfs_attestation.device_number != "259:3" || + btrfs_attestation.nvme_device != "nvme0n1") { + throw std::runtime_error("btrfs NVMe partition attestation mismatch"); + } + require_failure( + [&]() { + dsv41::require_nvme_path( + root / "tmp-link" / "child", "symlink traversal", mountinfo, + sys / "dev" / "block", sys / "class" / "block"); + }, + "must not be a symlink"); + require_failure( + [&]() { + dsv41::require_nvme_path( + rotating / "data", "rotating", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "non-rotational"); + require_failure( + [&]() { + dsv41::require_nvme_path( + ram / "data", "tmpfs", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "local block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + network / "data", "network", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "local block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + missing / "data", "missing", mountinfo, sys / "dev" / "block", sys / "class" / "block"); + }, + "resolvable block device"); + require_failure( + [&]() { + dsv41::require_nvme_path( + xfs / "escape" / "data", "symlink escape", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + }, + "must not be a symlink"); + require_failure( + [&]() { + dsv41::require_nvme_path( + "/mnt/bigspace/model.gguf", "forbidden", mountinfo, sys / "dev" / "block", + sys / "class" / "block"); + }, + "/mnt/bigspace"); + require_failure( + [&]() { + dsv41::require_nvme_path( + forbidden / "escape" / "model.gguf", "forbidden symlink", mountinfo, + sys / "dev" / "block", sys / "class" / "block", forbidden); + }, + "must not use"); + + const fs::path kfd = root / "kfd"; + write_file( + kfd / "1" / "properties", + "domain 0\nlocation_id 50688\ngfx_target_version 110501\n"); + write_file(kfd / "1" / "gpu_id", "1234\n"); + const dsv41::accelerator_attestation accelerator = + dsv41::require_gfx1151_identity( + "ROCm0", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + if (accelerator.architecture != "gfx1151" || accelerator.gfx_target_version != 110501) { + throw std::runtime_error("gfx1151 attestation mismatch"); + } + require_failure( + [&]() { + dsv41::require_gfx1151_identity( + "ROCm1", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + }, + "ROCm0"); + write_file( + kfd / "1" / "properties", + "domain 0\nlocation_id 50688\ngfx_target_version 110500\n"); + require_failure( + [&]() { + dsv41::require_gfx1151_identity( + "ROCm0", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + }, + "gfx1151"); + write_file( + kfd / "1" / "properties", + "domain 0\nlocation_id 50688\n"); + require_failure( + [&]() { + dsv41::require_gfx1151_identity( + "ROCm0", "AMD Radeon 8060S Graphics", "0000:c6:00.0", kfd); + }, + "exactly one KFD"); + + fs::remove_all(root); + return 0; + } catch (const std::exception & error) { + std::cerr << "test-host-attestation: " << error.what() << '\n'; + std::error_code ec; + fs::remove_all(root, ec); + return 1; + } +} diff --git a/tools/deepseek-v41-trace/test-injected-library.cpp b/tools/deepseek-v41-trace/test-injected-library.cpp new file mode 100644 index 000000000000..cd50087a639d --- /dev/null +++ b/tools/deepseek-v41-trace/test-injected-library.cpp @@ -0,0 +1,5 @@ +extern "C" int dsv41_test_injected_library(void); + +extern "C" int dsv41_test_injected_library(void) { + return 1; +} diff --git a/tools/deepseek-v41-trace/test-install.cmake b/tools/deepseek-v41-trace/test-install.cmake new file mode 100644 index 000000000000..57588eae2db9 --- /dev/null +++ b/tools/deepseek-v41-trace/test-install.cmake @@ -0,0 +1,99 @@ +if(NOT DEFINED DSV41_BUILD_DIR OR NOT DEFINED DSV41_INSTALL_ROOT OR + NOT DEFINED DSV41_COMPONENT OR NOT DEFINED DSV41_EXECUTABLE OR + NOT DEFINED DSV41_SOURCE_EXECUTABLE OR + NOT DEFINED DSV41_REVISION OR NOT DEFINED DSV41_PYTHON OR + NOT DEFINED DSV41_RECEIPT OR NOT DEFINED DSV41_CONTAINMENT_HELPER OR + NOT DEFINED DSV41_CONTAINMENT_HELPER_RECEIPT OR NOT DEFINED DSV41_VERIFY_SCRIPT) + message(FATAL_ERROR "DeepSeek V4.1 install smoke test arguments are incomplete") +endif() + +file(REMOVE_RECURSE "${DSV41_INSTALL_ROOT}") +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "${DSV41_BUILD_DIR}" + --prefix "${DSV41_INSTALL_ROOT}" + --component "${DSV41_COMPONENT}" + --config "${DSV41_CONFIG}" + RESULT_VARIABLE install_result + OUTPUT_VARIABLE install_output + ERROR_VARIABLE install_error) +if(NOT install_result EQUAL 0) + message(FATAL_ERROR "DeepSeek V4.1 component install failed:\n${install_output}${install_error}") +endif() + +file(SHA256 "${DSV41_SOURCE_EXECUTABLE}" source_executable_sha256) +file(SHA256 "${DSV41_INSTALL_ROOT}/bin/${DSV41_EXECUTABLE}" installed_executable_sha256) +if(NOT source_executable_sha256 STREQUAL installed_executable_sha256) + message(FATAL_ERROR "installed DeepSeek V4.1 trace executable differs from linked bytes") +endif() + +get_filename_component(containment_helper_name "${DSV41_CONTAINMENT_HELPER}" NAME) +file(SHA256 "${DSV41_CONTAINMENT_HELPER}" source_helper_sha256) +file(SHA256 "${DSV41_INSTALL_ROOT}/bin/${containment_helper_name}" installed_helper_sha256) +if(NOT source_helper_sha256 STREQUAL installed_helper_sha256) + message(FATAL_ERROR "installed DeepSeek V4.1 containment helper differs from linked bytes") +endif() +file( + SHA256 "${DSV41_CONTAINMENT_HELPER_RECEIPT}" + source_containment_helper_receipt_sha256) +file( + SHA256 + "${DSV41_INSTALL_ROOT}/share/deepseek-v41-trace/dsv41-containment-helper-receipt.json" + installed_containment_helper_receipt_sha256) +if(NOT source_containment_helper_receipt_sha256 STREQUAL + installed_containment_helper_receipt_sha256) + message(FATAL_ERROR "installed DeepSeek V4.1 containment helper receipt differs from build bytes") +endif() + +execute_process( + COMMAND "${DSV41_PYTHON}" "${DSV41_VERIFY_SCRIPT}" + --receipt "${DSV41_RECEIPT}" + --install-root "${DSV41_INSTALL_ROOT}" + RESULT_VARIABLE receipt_result + OUTPUT_VARIABLE receipt_output + ERROR_VARIABLE receipt_error) +if(NOT receipt_result EQUAL 0) + message(FATAL_ERROR + "installed DeepSeek V4.1 runtime receipt differs from linked bytes:\n${receipt_output}${receipt_error}") +endif() + +unset(ENV{DYLD_FALLBACK_LIBRARY_PATH}) +unset(ENV{DYLD_FALLBACK_FRAMEWORK_PATH}) +unset(ENV{DYLD_FRAMEWORK_PATH}) +unset(ENV{DYLD_IMAGE_SUFFIX}) +unset(ENV{DYLD_INSERT_LIBRARIES}) +unset(ENV{DYLD_LIBRARY_PATH}) +unset(ENV{DYLD_ROOT_PATH}) +unset(ENV{DYLD_VERSIONED_FRAMEWORK_PATH}) +unset(ENV{DYLD_VERSIONED_LIBRARY_PATH}) +unset(ENV{GGML_BACKEND_PATH}) +unset(ENV{LD_LIBRARY_PATH}) +unset(ENV{LD_PRELOAD}) +unset(ENV{PATH}) + +execute_process( + COMMAND "${DSV41_INSTALL_ROOT}/bin/${DSV41_EXECUTABLE}" --version + RESULT_VARIABLE smoke_result + OUTPUT_VARIABLE smoke_output + ERROR_VARIABLE smoke_error) +if(NOT smoke_result EQUAL 0) + message(FATAL_ERROR "installed DeepSeek V4.1 trace --version failed:\n${smoke_output}${smoke_error}") +endif() +if(NOT smoke_output MATCHES "commit ${DSV41_REVISION}") + message(FATAL_ERROR "installed DeepSeek V4.1 trace reported the wrong revision:\n${smoke_output}") +endif() + +execute_process( + COMMAND "${DSV41_INSTALL_ROOT}/bin/${containment_helper_name}" --version + RESULT_VARIABLE helper_smoke_result + OUTPUT_VARIABLE helper_smoke_output + ERROR_VARIABLE helper_smoke_error) +if(NOT helper_smoke_result EQUAL 0) + message(FATAL_ERROR + "installed DeepSeek V4.1 containment helper --version failed:\n" + "${helper_smoke_output}${helper_smoke_error}") +endif() +if(NOT helper_smoke_output MATCHES "${DSV41_REVISION}") + message(FATAL_ERROR + "installed DeepSeek V4.1 containment helper reported the wrong revision:\n" + "${helper_smoke_output}") +endif() diff --git a/tools/deepseek-v41-trace/trace-components.h b/tools/deepseek-v41-trace/trace-components.h new file mode 100644 index 000000000000..4e88f269288c --- /dev/null +++ b/tools/deepseek-v41-trace/trace-components.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include + +struct dsv41_trace_descriptor { + std::string component; + int layer; + const char * semantic_id_space; +}; + +inline bool dsv41_trace_expected_layer(const std::string & component, int layer) { + if (component == "engram.row_ids") { + return layer == 1 || layer == 14; + } + if (component == "expert.ids" || component == "expert.weights" || component == "attn.source") { + return layer >= 0 && layer < 40; + } + if (component == "attn.candidate_blocks") { + return layer == 20; + } + if (component == "attn.candidates") { + return layer == 24 || layer == 28 || layer == 32 || layer == 36; + } + return false; +} + +inline std::optional dsv41_trace_parse_name(const std::string & name) { + struct prefix_entry { + const char * prefix; + const char * component; + const char * semantic_id_space; + }; + static const prefix_entry prefixes[] = { + {"dsv41.trace.engram.row_ids.l", "engram.row_ids", nullptr}, + {"dsv41.trace.expert.ids.l", "expert.ids", "original"}, + {"dsv41.trace.expert.weights.l", "expert.weights", nullptr}, + {"dsv41.trace.attn.source.l", "attn.source", nullptr}, + {"dsv41.trace.attn.candidate_blocks.l", "attn.candidate_blocks", nullptr}, + {"dsv41.trace.attn.candidates.l", "attn.candidates", nullptr}, + }; + for (const prefix_entry & entry : prefixes) { + const std::string prefix = entry.prefix; + if (name.rfind(prefix, 0) != 0 || name.size() == prefix.size()) { + continue; + } + int layer = 0; + for (size_t index = prefix.size(); index < name.size(); ++index) { + const char value = name[index]; + if (value < '0' || value > '9') { + throw std::runtime_error("malformed reserved trace tensor name: " + name); + } + layer = 10*layer + value - '0'; + if (layer >= 40) { + throw std::runtime_error("unexpected reserved trace tensor layer: " + name); + } + } + if (!dsv41_trace_expected_layer(entry.component, layer)) { + throw std::runtime_error("unexpected reserved trace tensor layer: " + name); + } + return dsv41_trace_descriptor{entry.component, layer, entry.semantic_id_space}; + } + if (name.rfind("dsv41.trace.", 0) == 0) { + throw std::runtime_error("malformed reserved trace tensor name: " + name); + } + return std::nullopt; +} + +inline bool dsv41_trace_select_name(const std::string & name) { + return dsv41_trace_parse_name(name).has_value(); +} diff --git a/tools/deepseek-v41-trace/trace_format.py b/tools/deepseek-v41-trace/trace_format.py new file mode 100644 index 000000000000..de0b7988d967 --- /dev/null +++ b/tools/deepseek-v41-trace/trace_format.py @@ -0,0 +1,6432 @@ +#!/usr/bin/env python3 + +import argparse +import array +import ctypes +import hashlib +import json +import math +import os +import re +import select +import selectors +import signal +import socket +import stat +import struct +import subprocess +import sys +import tempfile +import threading +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterable + +TRACE_FORMAT = "dsv41-trace" +TRACE_VERSION = 2 +DS4_REVISION = "bd66c402070042bf0a79ad6ece8242de4c93680c" +DS4_REPOSITORY = "antirez/ds4" +APPROVED_TRACE_SIGNERS: dict[str, dict[str, str]] = {} +APPROVED_EXECUTABLE_APPROVERS: dict[str, dict[str, Any]] = {} +APPROVED_CANDIDATE_EXPORTERS: dict[str, dict[str, Any]] = {} +APPROVED_DS4_EXPORTERS: dict[str, dict[str, Any]] = {} +APPROVED_PROMPT_BUILDERS: dict[str, dict[str, Any]] = {} +EXECUTABLE_APPROVAL_FORMAT = "dsv41-executable-approval" +EXECUTABLE_APPROVAL_VERSION = 2 +EXECUTABLE_APPROVAL_NAMESPACE = "dsv41-executable-approval-v2" +SEAL_FORMAT = "dsv41-trace-bundle-signature" +SEAL_VERSION = 1 +SEAL_NAMESPACE = "dsv41-trace-bundle-v1" +SEAL_DOMAIN_PREFIX = b"dsv41-trace-bundle-v1\n" +SIGNATURE_NAME = "bundle-signature.json" +CANDIDATE_LANE = "strix-llama-candidate-v1" +ORACLE_LANE = "apple-ds4-oracle-v1" +AUTHORIZATION_FORMAT = "dsv41-execution-authorization" +AUTHORIZATION_VERSION = 1 +MAX_AUTHORIZATION_LIFETIME_SECONDS = 24 * 60 * 60 +MODEL_SHA256 = "1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42" +REPOSITORY = "halo-box/strix-llama.cpp" +SOFT_MEMORY_LIMIT = 116 * 1024 * 1024 * 1024 +WATCHDOG_EMERGENCY_LIMIT = 118 * 1024 * 1024 * 1024 +STRICT_MEMORY_LIMIT = 120 * 1024 * 1024 * 1024 +WATCHDOG_LEASE_FORMAT = "strix-memory-watchdog-lease" +WATCHDOG_VERSION = 2 +WATCHDOG_REVISION = "778db6f50eae04e6c232c69b9575bdbd0747962b" +WATCHDOG_SCRIPT_SHA256 = "d2781a25f978dd2bc14fc113079aa2dbf513aa157b44da9d0d51d750daa6c94f" +APPROVED_WATCHDOGS = {WATCHDOG_SCRIPT_SHA256: WATCHDOG_REVISION} +NO_EXTERNAL_STATE_STORAGE = { + "format": "dsv41-state-storage-policy", + "version": 1, + "expert_cache": "memory-resident", + "kv_cache": "memory-resident", + "external_cache_paths": [], + "external_state_paths": [], +} +ADMITTED_UBATCH = 32 +ADMITTED_BATCH = 2048 +EXPERT_COUNT = 384 +EXPERTS_USED = 6 +EXPERT_SLOT_BYTES = 398_131_200 +REQUIRED_EXPERT_SLOTS = min(EXPERT_COUNT, EXPERTS_USED * ADMITTED_UBATCH) +REQUIRED_EXPERT_CACHE_BYTES = REQUIRED_EXPERT_SLOTS * EXPERT_SLOT_BYTES +REQUIRED_EXPERT_CACHE_MIB = REQUIRED_EXPERT_CACHE_BYTES // (1024 * 1024) +RAW_ATTENTION_LAYERS = (0, 1) +RAW_ATTENTION_WIDTH = 128 +CORPUS_SHA256 = { + "correctness-prose.txt": "2da590a37e3297767336c10b024a0de732d64bee4da5792596f8ddf49ea408d2", + "correctness-code.txt": "41b4246ef4e6b4e3f9f23a3d02aa8cdab48f495b3af0ebeaccea255679c771f0", + "correctness-structured.txt": "1278707adea5a953196c4cf5c04de301952813be3eac416a6aac4ff94f42f701", + "correctness-numeric.txt": "ebd444cf70662cc09289af45ef654af0953b98e967a449d031627d8ea92bc2e0", +} +MANIFEST_NAME = "manifest.json" +EVENTS_NAME = "events.jsonl" +BLOBS_DIR = "blobs" +FORBIDDEN_LOADER_ENVIRONMENT = ( + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_IMAGE_SUFFIX", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + "DYLD_ROOT_PATH", + "DYLD_VERSIONED_FRAMEWORK_PATH", + "DYLD_VERSIONED_LIBRARY_PATH", + "GGML_BACKEND_PATH", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", +) + +DTYPE_SIZES = { + "f32": 4, + "bf16": 2, + "i32": 4, + "u32": 4, + "i8": 1, + "u8": 1, + "bytes": 1, +} + +HARD_FAILURE_COMPONENTS = ( + "prompt.bytes", + "prompt.tokens", + "engram.row_ids", + "expert.ids", + "expert.weights", + "attn.source", + "attn.candidate_blocks", + "attn.candidates", + "logits.prefill", + "logits.decode", + "decode.greedy_token", +) + +DEEPSEEK41_EXPECTED_COMPONENTS = { + "prompt.bytes": {"layers": None, "input": "tokens"}, + "prompt.tokens": {"layers": None, "input": "tokens"}, + "engram.row_ids": {"layers": [1, 14], "prefill": "tokens", "decode": "steps"}, + "expert.ids": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "expert.weights": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.source": {"layers": list(range(40)), "prefill": "tokens", "decode": "steps"}, + "attn.candidate_blocks": {"layers": [20], "prefill": "tokens", "decode": "steps"}, + "attn.candidates": {"layers": [24, 28, 32, 36], "prefill": "tokens", "decode": "steps"}, + "logits.prefill": {"layers": None, "prefill": "final"}, + "logits.decode": {"layers": None, "decode": "steps"}, + "decode.greedy_token": {"layers": None, "decode": "steps"}, +} + + +class TraceError(RuntimeError): + pass + + +PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS = 5 +PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES = 65536 +LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS = 8 +PROCESS_TREE_TERM_GRACE_SECONDS = 1 +WINDOWS_CREATE_SUSPENDED = 0x00000004 +WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 + + +@dataclass(frozen=True) +class _IntegrityFailure: + component: str + error: BaseException + + +class ExecutionIntegrityError(TraceError): + def __init__( + self, + message: str, + *, + primary_error: BaseException | None, + secondary_errors: list[_IntegrityFailure], + quiescence_proven: bool = False): + super().__init__(message) + self.primary_error = primary_error + self.secondary_errors = tuple(secondary_errors) + self.quiescence_proven = quiescence_proven + + +@dataclass +class _ProcessContainment: + process: Any + linux_root_pidfd: int | None = None + linux_namespace_pidfd: int | None = None + linux_lock_held: bool = False + linux_exec_released: bool = False + test_process_group_id: int | None = None + job_handle: int | None = None + windows_job_assigned: bool = False + windows_process_resumed: bool = False + + +@dataclass +class _ContainedRun: + result: subprocess.CompletedProcess[bytes] | None + primary_error: BaseException | None + integrity_failures: list[_IntegrityFailure] + containment: _ProcessContainment | None + quiescence_proven: bool = False + process_started: bool = False + + +@dataclass +class _ContainmentCleanup: + failures: list[_IntegrityFailure] + quiescence_proven: bool + + +_LINUX_HELPER_LOCK = threading.Lock() +_LINUX_HELPER_POISONED = False +_LINUX_POISONED_CONTAINMENT: _ProcessContainment | None = None +_TEST_PROCESS_GROUP_CONTAINMENT = threading.local() + + +@dataclass(frozen=True) +class TraceVerifier: + principal: str + trusted_signers: dict[str, dict[str, str]] + ssh_keygen: Path + expected_lane: str + expected_challenge: str + expected_run_id: str + verification_unix: int + candidate_exporter_policies: dict[str, dict[str, Any]] + ds4_exporter_policies: dict[str, dict[str, Any]] + prompt_builder_policies: dict[str, dict[str, Any]] + expected_candidate_exporter_policy_id: str | None + expected_ds4_exporter_policy_id: str | None + expected_prompt_builder_policy_id: str + expected_approval_policy_sha256: str + expected_verifier_revision: str + seen_run_ids: set[str] | None = None + test_only: bool = False + + @classmethod + def production( + cls, + principal: str, + *, + expected_lane: str, + expected_challenge: str, + expected_run_id: str, + expected_candidate_exporter_policy_id: str | None, + expected_ds4_exporter_policy_id: str | None, + expected_prompt_builder_policy_id: str, + approval_policy: "ExecutableApprovalPolicy | None" = None, + verification_unix: int | None, + seen_run_ids: set[str] | None = None) -> "TraceVerifier": + return cls( + principal=principal, + trusted_signers=APPROVED_TRACE_SIGNERS, + ssh_keygen=trusted_ssh_keygen_path(), + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=int(time.time()) if verification_unix is None else verification_unix, + candidate_exporter_policies=( + approval_policy.candidate_exporters + if approval_policy is not None else APPROVED_CANDIDATE_EXPORTERS), + ds4_exporter_policies=( + approval_policy.ds4_exporters + if approval_policy is not None else APPROVED_DS4_EXPORTERS), + prompt_builder_policies=( + approval_policy.prompt_builders + if approval_policy is not None else APPROVED_PROMPT_BUILDERS), + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=( + approval_policy.sha256 if approval_policy is not None else ""), + expected_verifier_revision=( + approval_policy.verifier_revision if approval_policy is not None else ""), + seen_run_ids=seen_run_ids, + ) + + @classmethod + def for_tests( + cls, + principal: str, + public_key: str, + *, + lane: str, + runtime: str, + runtime_profile: str, + expected_challenge: str, + expected_run_id: str, + candidate_exporter_policies: dict[str, dict[str, Any]] | None = None, + ds4_exporter_policies: dict[str, dict[str, Any]] | None = None, + prompt_builder_policies: dict[str, dict[str, Any]] | None = None, + expected_candidate_exporter_policy_id: str | None = None, + expected_ds4_exporter_policy_id: str | None = None, + expected_prompt_builder_policy_id: str = "", + expected_approval_policy_sha256: str = "e" * 64, + expected_verifier_revision: str = "a" * 40, + verification_unix: int | None = None, + ssh_keygen: Path | None = None, + seen_run_ids: set[str] | None = None) -> "TraceVerifier": + return cls( + principal=principal, + trusted_signers={ + principal: { + "public_key": public_key, + "lane": lane, + "runtime": runtime, + "runtime_profile": runtime_profile, + }, + }, + ssh_keygen=ssh_keygen or trusted_ssh_keygen_path(), + expected_lane=lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=verification_unix, + candidate_exporter_policies=candidate_exporter_policies or {}, + ds4_exporter_policies=ds4_exporter_policies or {}, + prompt_builder_policies=prompt_builder_policies or {}, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=expected_approval_policy_sha256, + expected_verifier_revision=expected_verifier_revision, + seen_run_ids=seen_run_ids, + test_only=True, + ) + + +@dataclass(frozen=True) +class BundleFileReceipt: + device: int + inode: int + byte_count: int + modified_ns: int + changed_ns: int + sha256: str + + +@dataclass(frozen=True) +class ExecutableFileReceipt: + path: str + install_root: str + device: int + inode: int + owner_uid: int + mode: int + link_count: int + byte_count: int + modified_ns: int + changed_ns: int + sha256: str + path_chain: tuple[tuple[str, int, int, int, int], ...] + + +@dataclass(frozen=True) +class ExecutableApprovalPolicy: + principal: str + verifier_revision: str + candidate_exporters: dict[str, dict[str, Any]] + ds4_exporters: dict[str, dict[str, Any]] + prompt_builders: dict[str, dict[str, Any]] + sha256: str + + +@dataclass(frozen=True) +class Mismatch: + classification: str + component: str + phase: str + step: int + token_start: int + layer: int | None + detail: str + element_index: int | None = None + byte_offset: int | None = None + token_index: int | None = None + component_element_index: int | None = None + + def as_dict(self) -> dict[str, Any]: + result = { + "classification": self.classification, + "component": self.component, + "phase": self.phase, + "step": self.step, + "token_start": self.token_start, + "layer": self.layer, + "detail": self.detail, + } + if self.element_index is not None: + result["element_index"] = self.element_index + if self.byte_offset is not None: + result["byte_offset"] = self.byte_offset + if self.token_index is not None: + result["token_index"] = self.token_index + if self.component_element_index is not None: + result["component_element_index"] = self.component_element_index + return result + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _execution_uid() -> int: + if os.name != "posix" or not hasattr(os, "geteuid"): + raise TraceError("immutable install paths require a POSIX execution identity") + return os.geteuid() + + +def _path_is_writable_by_execution_identity(path: Path) -> bool: + if _execution_uid() == 0: + raise TraceError("immutable install paths require an unprivileged POSIX execution identity") + try: + return os.access(path, os.W_OK, effective_ids=True) + except (OSError, TypeError, NotImplementedError) as error: + raise TraceError(f"cannot verify effective write access for {path}: {error}") from error + + +def _require_distinct_trusted_owner(expected_owner_uid: int) -> None: + execution_uid = _execution_uid() + if execution_uid == 0 or expected_owner_uid == execution_uid: + raise TraceError("approved install tree owner must be distinct from the unprivileged execution identity") + + +def _has_access_control_entries(path: Path) -> bool: + if sys.platform.startswith("linux"): + try: + attributes = os.listxattr(path, follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect access controls for {path}: {error}") from error + return any(name in {"system.posix_acl_access", "system.posix_acl_default"} for name in attributes) + if sys.platform == "darwin": + tool = Path("/bin/ls") + try: + tool_stat = tool.stat(follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect the fixed macOS ACL verifier: {error}") from error + if not stat.S_ISREG(tool_stat.st_mode) or tool_stat.st_uid != 0 or ( + stat.S_IMODE(tool_stat.st_mode) & 0o022): + raise TraceError("the fixed macOS ACL verifier is not trusted") + try: + result = subprocess.run( + [str(tool), "-lde", str(path)], + stdin=subprocess.DEVNULL, + check=False, + capture_output=True, + text=True, + timeout=30, + env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot inspect access controls for {path}: {error}") from error + if result.returncode != 0: + raise TraceError(f"cannot inspect access controls for {path}") + first_line = result.stdout.splitlines()[0] if result.stdout else "" + fields = first_line.split() + if not fields or len(fields[0]) < 10: + raise TraceError(f"access control output for {path} is invalid") + return "+" in fields[0] + raise TraceError("immutable install path access-control verification is unsupported on this platform") + + +def _immutable_path_chain( + path: Path, + *, + install_root: Path, + expected_owner_uid: int, + label: str, +) -> tuple[tuple[str, int, int, int, int], ...]: + if not path.is_absolute() or not install_root.is_absolute() or ( + path != install_root and install_root not in path.parents): + raise TraceError(f"{label} path is outside its approved install root") + try: + if str(path.resolve(strict=True)) != str(path) or str(install_root.resolve(strict=True)) != str(install_root): + raise TraceError(f"{label} path must be canonical and must not use aliases") + except OSError as error: + raise TraceError(f"cannot resolve {label} path: {error}") from error + if type(expected_owner_uid) is not int or expected_owner_uid < 0: + raise TraceError(f"{label} install owner is invalid") + _require_distinct_trusted_owner(expected_owner_uid) + result = [] + current = Path(path.anchor) + candidates = [current] + for part in path.parent.parts[1:]: + current /= part + candidates.append(current) + install_seen = Path(path.anchor) == install_root + for current in candidates: + try: + current_stat = current.stat(follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect {label} path: {error}") from error + if stat.S_ISLNK(current_stat.st_mode): + raise TraceError(f"{label} path must not use symlinks") + if not stat.S_ISDIR(current_stat.st_mode): + raise TraceError(f"{label} parent path is not a directory") + if current == install_root: + install_seen = True + if current_stat.st_uid not in {0, expected_owner_uid}: + raise TraceError(f"{label} path is not trusted-owned") + if install_seen and current_stat.st_uid != expected_owner_uid: + raise TraceError(f"{label} install tree owner differs from external approval") + if stat.S_IMODE(current_stat.st_mode) & 0o022 or ( + _path_is_writable_by_execution_identity(current)) or _has_access_control_entries(current): + raise TraceError(f"{label} path is mutable by the execution identity or an untrusted group") + result.append(( + str(current), + current_stat.st_dev, + current_stat.st_ino, + current_stat.st_uid, + stat.S_IMODE(current_stat.st_mode), + )) + if not install_seen: + raise TraceError(f"{label} path does not traverse its approved install root") + return tuple(result) + + +def _approved_file_identity( + path: Path, + *, + install_root: Path, + expected_owner_uid: int, + expected_path: str, + expected_sha256: str, + label: str, + executable: bool, +) -> tuple[ExecutableFileReceipt, int]: + if not path.is_absolute() or str(path) != expected_path or not install_root.is_absolute(): + raise TraceError(f"{label} path differs from external approval") + path_chain = _immutable_path_chain( + path, + install_root=install_root, + expected_owner_uid=expected_owner_uid, + label=label, + ) + flags = os.O_RDONLY + flags |= getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + except OSError as error: + raise TraceError(f"cannot open {label}: {error}") from error + if not stat.S_ISREG(before.st_mode) or before.st_uid != expected_owner_uid or before.st_nlink != 1 or ( + stat.S_IMODE(before.st_mode) & 0o222) or ( + _path_is_writable_by_execution_identity(path)) or _has_access_control_entries(path) or ( + executable and not (stat.S_IMODE(before.st_mode) & 0o111)): + os.close(descriptor) + raise TraceError(f"{label} is not an immutable trusted-owned one-link regular file") + try: + digest = hashlib.sha256() + with os.fdopen(os.dup(descriptor), "rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + digest_value = digest.hexdigest() + after = os.fstat(descriptor) + path_after = path.stat(follow_symlinks=False) + except OSError as error: + os.close(descriptor) + raise TraceError(f"cannot recheck {label}: {error}") from error + identity = ( + before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) + if identity != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) or identity != ( + path_after.st_dev, path_after.st_ino, path_after.st_size, + path_after.st_mtime_ns, path_after.st_ctime_ns): + os.close(descriptor) + raise TraceError(f"{label} changed while hashing") + if digest_value != expected_sha256: + os.close(descriptor) + raise TraceError(f"{label} SHA-256 differs from external approval") + return ExecutableFileReceipt( + path=str(path), + install_root=str(install_root), + device=after.st_dev, + inode=after.st_ino, + owner_uid=after.st_uid, + mode=stat.S_IMODE(after.st_mode), + link_count=after.st_nlink, + byte_count=after.st_size, + modified_ns=after.st_mtime_ns, + changed_ns=after.st_ctime_ns, + sha256=digest_value, + path_chain=path_chain, + ), descriptor + + +def approved_executable_identity( + path: Path, + *, + install_root: str, + expected_owner_uid: int, + expected_path: str, + expected_sha256: str, + label: str, +) -> ExecutableFileReceipt: + identity, descriptor = _approved_file_identity( + path, + install_root=Path(install_root), + expected_owner_uid=expected_owner_uid, + expected_path=expected_path, + expected_sha256=expected_sha256, + label=label, + executable=True, + ) + os.close(descriptor) + return identity + + +def verify_approved_executable_identity( + path: Path, + expected: ExecutableFileReceipt, + *, + label: str, +) -> None: + observed = approved_executable_identity( + path, + install_root=expected.install_root, + expected_owner_uid=expected.owner_uid, + expected_path=expected.path, + expected_sha256=expected.sha256, + label=label, + ) + if observed != expected: + raise TraceError(f"{label} identity changed after approval") + + +def approved_runtime_file_identities( + policy: dict[str, Any], + *, + label: str, +) -> list[ExecutableFileReceipt]: + install_root = Path(policy["install_root"]) + identities = [] + for component in policy["runtime_receipt"]["components"]: + path = install_root / "lib" / component["filename"] + identity, descriptor = _approved_file_identity( + path, + install_root=install_root, + expected_owner_uid=policy["install_owner_uid"], + expected_path=str(path), + expected_sha256=component["sha256"], + label=f"{label} runtime component {component['component']}", + executable=False, + ) + os.close(descriptor) + identities.append(identity) + return identities + + +def verify_approved_runtime_file_identities( + identities: list[ExecutableFileReceipt], + *, + label: str, +) -> None: + for identity in identities: + observed, descriptor = _approved_file_identity( + Path(identity.path), + install_root=Path(identity.path).parent.parent, + expected_owner_uid=identity.owner_uid, + expected_path=identity.path, + expected_sha256=identity.sha256, + label=f"{label} runtime component", + executable=False, + ) + os.close(descriptor) + if observed != identity: + raise TraceError(f"{label} runtime component identity changed after approval") + + +def _windows_error(message: str) -> TraceError: + return TraceError(f"{message}: Windows error {ctypes.get_last_error()}") + + +def _windows_kernel32() -> Any: + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR] + kernel32.CreateJobObjectW.restype = wintypes.HANDLE + kernel32.SetInformationJobObject.argtypes = [ + wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD] + kernel32.SetInformationJobObject.restype = wintypes.BOOL + kernel32.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE] + kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + kernel32.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT] + kernel32.TerminateJobObject.restype = wintypes.BOOL + kernel32.QueryInformationJobObject.argtypes = [ + wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD, ctypes.c_void_p] + kernel32.QueryInformationJobObject.restype = wintypes.BOOL + kernel32.CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD] + kernel32.CreateToolhelp32Snapshot.restype = wintypes.HANDLE + kernel32.Thread32First.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + kernel32.Thread32First.restype = wintypes.BOOL + kernel32.Thread32Next.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + kernel32.Thread32Next.restype = wintypes.BOOL + kernel32.OpenThread.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel32.OpenThread.restype = wintypes.HANDLE + kernel32.ResumeThread.argtypes = [wintypes.HANDLE] + kernel32.ResumeThread.restype = wintypes.DWORD + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + return kernel32 + + +def _create_windows_kill_job() -> int: + from ctypes import wintypes + + class BasicLimitInformation(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_int64), + ("PerJobUserTimeLimit", ctypes.c_int64), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class IoCounters(ctypes.Structure): + _fields_ = [ + ("ReadOperationCount", ctypes.c_uint64), + ("WriteOperationCount", ctypes.c_uint64), + ("OtherOperationCount", ctypes.c_uint64), + ("ReadTransferCount", ctypes.c_uint64), + ("WriteTransferCount", ctypes.c_uint64), + ("OtherTransferCount", ctypes.c_uint64), + ] + + class ExtendedLimitInformation(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", BasicLimitInformation), + ("IoInfo", IoCounters), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + kernel32 = _windows_kernel32() + job = kernel32.CreateJobObjectW(None, None) + if not job: + raise _windows_error("cannot create process containment job") + limits = ExtendedLimitInformation() + limits.BasicLimitInformation.LimitFlags = WINDOWS_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if not kernel32.SetInformationJobObject( + job, 9, ctypes.byref(limits), ctypes.sizeof(limits)): + primary_error = _windows_error("cannot configure process containment job") + if not kernel32.CloseHandle(job): + close_error = _windows_error("cannot close unconfigured process containment job") + raise ExecutionIntegrityError( + f"Windows job configuration primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: windows-job-handle-close " + f"[{type(close_error).__name__}: {close_error}]", + primary_error=primary_error, + secondary_errors=[_IntegrityFailure("windows-job-handle-close", close_error)], + ) from primary_error + raise primary_error + return int(job) + + +def _open_windows_process_thread(process_id: int) -> int: + from ctypes import wintypes + + class ThreadEntry32(ctypes.Structure): + _fields_ = [ + ("dwSize", wintypes.DWORD), + ("cntUsage", wintypes.DWORD), + ("th32ThreadID", wintypes.DWORD), + ("th32OwnerProcessID", wintypes.DWORD), + ("tpBasePri", wintypes.LONG), + ("tpDeltaPri", wintypes.LONG), + ("dwFlags", wintypes.DWORD), + ] + + kernel32 = _windows_kernel32() + snapshot = kernel32.CreateToolhelp32Snapshot(0x00000004, 0) + if not snapshot or int(snapshot) == ctypes.c_void_p(-1).value: + raise _windows_error("cannot enumerate suspended process threads") + entry = ThreadEntry32() + entry.dwSize = ctypes.sizeof(entry) + thread = None + try: + found = kernel32.Thread32First(snapshot, ctypes.byref(entry)) + while found: + if entry.th32OwnerProcessID == process_id: + thread = kernel32.OpenThread(0x0002 | 0x00100000, False, entry.th32ThreadID) + if not thread: + raise _windows_error("cannot open suspended process thread") + break + found = kernel32.Thread32Next(snapshot, ctypes.byref(entry)) + except BaseException as primary_error: + if not kernel32.CloseHandle(snapshot): + close_error = _windows_error("cannot close process thread snapshot") + raise ExecutionIntegrityError( + f"Windows thread enumeration primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: windows-snapshot-handle-close " + f"[{type(close_error).__name__}: {close_error}]", + primary_error=primary_error, + secondary_errors=[_IntegrityFailure("windows-snapshot-handle-close", close_error)], + ) from primary_error + raise + if not kernel32.CloseHandle(snapshot): + close_error = _windows_error("cannot close process thread snapshot") + failures = [_IntegrityFailure("windows-snapshot-handle-close", close_error)] + if thread: + if not kernel32.CloseHandle(thread): + failures.append(_IntegrityFailure( + "windows-thread-handle-close", + _windows_error("cannot close suspended process thread"))) + if len(failures) > 1: + raise ExecutionIntegrityError( + f"Windows thread enumeration integrity failures: " + f"{_format_integrity_failures(failures)}", + primary_error=None, + secondary_errors=failures, + ) from close_error + raise close_error + if not thread: + raise TraceError("cannot find suspended process thread") + return int(thread) + + +def _start_windows_job_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + creationflags = int(launch.pop("creationflags", 0)) | WINDOWS_CREATE_SUSPENDED + process = subprocess.Popen(command, creationflags=creationflags, **launch) + job = None + thread = None + job_assigned = False + process_resumed = False + try: + job = _create_windows_kill_job() + kernel32 = _windows_kernel32() + process_handle = int(process._handle) + if not kernel32.AssignProcessToJobObject(job, process_handle): + raise _windows_error("cannot assign suspended process to containment job") + job_assigned = True + thread = _open_windows_process_thread(process.pid) + if kernel32.ResumeThread(thread) == 0xFFFFFFFF: + raise _windows_error("cannot resume contained process") + process_resumed = True + thread_closed = kernel32.CloseHandle(thread) + thread = None + if not thread_closed: + raise _windows_error("cannot close resumed process thread") + return _ProcessContainment( + process=process, + job_handle=job, + windows_job_assigned=job_assigned, + windows_process_resumed=process_resumed, + ) + except BaseException as primary_error: + kernel32 = _windows_kernel32() + failures = [] + if thread is not None: + if not kernel32.CloseHandle(thread): + failures.append(_IntegrityFailure( + "windows-thread-handle-close", + _windows_error("cannot close suspended process thread"))) + if job_assigned and job is not None: + if not kernel32.TerminateJobObject(job, 1): + failures.append(_IntegrityFailure( + "windows-job-termination", + _windows_error("cannot terminate failed process containment job"))) + else: + try: + process.kill() + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-termination", error)) + try: + process.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-reap", error)) + try: + _close_windows_process_handle(process) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-handle-close", error)) + if job is not None and not kernel32.CloseHandle(job): + failures.append(_IntegrityFailure( + "windows-job-handle-close", + _windows_error("cannot close failed process containment job"))) + if failures: + raise ExecutionIntegrityError( + f"Windows containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]; " + f"secondary integrity failures: {_format_integrity_failures(failures)}", + primary_error=primary_error, + secondary_errors=failures, + quiescence_proven=False, + ) from primary_error + raise ExecutionIntegrityError( + f"Windows containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]", + primary_error=primary_error, + secondary_errors=[], + quiescence_proven=False, + ) from primary_error + + +@contextmanager +def _test_only_process_group_containment() -> Iterable[None]: + previous = getattr(_TEST_PROCESS_GROUP_CONTAINMENT, "enabled", False) + _TEST_PROCESS_GROUP_CONTAINMENT.enabled = True + try: + yield + finally: + _TEST_PROCESS_GROUP_CONTAINMENT.enabled = previous + + +def _linux_fd_is_close_on_exec(descriptor: int) -> bool: + import fcntl + + return bool(fcntl.fcntl(descriptor, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + + +class _LinuxNativeHelperProcess: + def __init__( + self, + command: list[str], + pid: int, + *, + protocol_socket: socket.socket, + stdin_fd: int | None, + stdout_fd: int | None, + stderr_fd: int | None): + self.args = command + self.pid = pid + self.returncode = None + self.root_pidfd = None + self.namespace_pidfd = None + self.completion_proven = False + self.launch_primary_error = None + self.launch_integrity_failures = [] + self._protocol_socket = protocol_socket + self._stdin_fd = stdin_fd + self._stdout_fd = stdout_fd + self._stderr_fd = stderr_fd + + def _receive_protocol(self, expected: bytes, *, receive_pidfd: bool = False) -> None: + item_size = array.array("i").itemsize + requested_flags = getattr(socket, "MSG_CMSG_CLOEXEC", 0) + data, ancillary, flags, _address = self._protocol_socket.recvmsg( + 128, + socket.CMSG_SPACE(item_size * LINUX_PROTOCOL_MAX_RECEIVED_DESCRIPTORS), + requested_flags, + ) + received = [] + rights_records = 0 + ancillary_error = None + for level, kind, content in ancillary: + if level != socket.SOL_SOCKET or kind != socket.SCM_RIGHTS: + ancillary_error = "Linux containment helper sent unexpected ancillary data" + continue + rights_records += 1 + complete_size = len(content) - len(content) % item_size + descriptor_bytes = array.array("i") + descriptor_bytes.frombytes(content[:complete_size]) + received.extend(descriptor_bytes) + if len(content) == 0 or complete_size != len(content): + ancillary_error = "Linux containment helper sent malformed descriptor data" + + def reject(message: str) -> None: + failures = [] + for descriptor in received: + try: + os.close(descriptor) + except BaseException as error: + failures.append(_IntegrityFailure( + "linux-helper-received-fd-close", error)) + primary = TraceError(message) + if failures: + raise ExecutionIntegrityError( + f"{message}; secondary integrity failures: " + f"{_format_integrity_failures(failures)}", + primary_error=primary, + secondary_errors=failures, + quiescence_proven=False, + ) from primary + raise primary + + allowed_flags = {0, requested_flags} + if flags not in allowed_flags: + reject( + f"Linux containment helper protocol returned unexpected flags {flags}") + if data != expected: + reject( + f"Linux containment helper protocol expected {expected.decode('ascii')}") + if ancillary_error is not None: + reject(ancillary_error) + if rights_records > 1: + reject("Linux containment helper sent multiple descriptor records") + if receive_pidfd: + if len(received) != 1: + reject("Linux containment helper did not provide one namespace pidfd") + descriptor = received[0] + try: + close_on_exec = _linux_fd_is_close_on_exec(descriptor) + except BaseException as error: + failures = [] + try: + os.close(descriptor) + except BaseException as close_error: + failures.append(_IntegrityFailure( + "linux-helper-received-fd-close", close_error)) + raise ExecutionIntegrityError( + f"Linux containment helper namespace pidfd validation failed " + f"[{type(error).__name__}: {error}]" + + (f"; secondary integrity failures: " + f"{_format_integrity_failures(failures)}" if failures else ""), + primary_error=error, + secondary_errors=failures, + quiescence_proven=False, + ) from error + if not close_on_exec: + reject("Linux containment helper namespace pidfd is not close-on-exec") + self.namespace_pidfd = descriptor + elif received: + reject("Linux containment helper sent an unexpected descriptor") + + def release_exec(self) -> None: + self._receive_protocol(b"READY") + self._protocol_socket.sendall(b"PREPARE") + self._receive_protocol(b"PREPARED", receive_pidfd=True) + self._protocol_socket.sendall(b"EXEC") + self._receive_protocol(b"RELEASED") + + def abort_blocked(self) -> _ContainmentCleanup: + failures = [] + try: + self._protocol_socket.shutdown(socket.SHUT_RDWR) + except BaseException as error: + failures.append(_IntegrityFailure("linux-helper-protocol-shutdown", error)) + try: + self._protocol_socket.close() + except BaseException as close_error: + failures.append(_IntegrityFailure( + "linux-process-fd-close:protocol", close_error)) + self._protocol_socket = None + try: + self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + except BaseException as error: + failures.append(_IntegrityFailure("linux-blocked-child-reap", error)) + if self._protocol_socket is not None: + try: + self._protocol_socket.close() + except BaseException as close_error: + failures.append(_IntegrityFailure( + "linux-process-fd-close:protocol", close_error)) + self._protocol_socket = None + try: + self.wait(timeout=PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + except BaseException as retry_error: + failures.append(_IntegrityFailure("linux-blocked-child-reap-retry", retry_error)) + return _ContainmentCleanup(failures, not failures and self.returncode is not None) + + def close_streams(self) -> list[_IntegrityFailure]: + failures = [] + if self._protocol_socket is not None: + try: + self._protocol_socket.close() + except BaseException as error: + failures.append(_IntegrityFailure("linux-process-fd-close:protocol", error)) + self._protocol_socket = None + for attribute in ("_stdin_fd", "_stdout_fd", "_stderr_fd"): + descriptor = getattr(self, attribute) + if descriptor is None: + continue + try: + os.close(descriptor) + except BaseException as error: + failures.append(_IntegrityFailure( + f"linux-process-fd-close:{attribute.removeprefix('_').removesuffix('_fd')}", + error, + )) + setattr(self, attribute, None) + return failures + + def collect_startup_stderr(self) -> tuple[bytes, list[_IntegrityFailure]]: + if self._stderr_fd is None: + return b"", [] + descriptor = self._stderr_fd + self._stderr_fd = None + data = bytearray() + failures = [] + try: + while len(data) <= PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES: + chunk = os.read( + descriptor, + PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES + 1 - len(data), + ) + if not chunk: + break + data.extend(chunk) + if len(data) > PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES: + failures.append(_IntegrityFailure( + "linux-helper-stderr-bounds", + TraceError("Linux helper startup stderr exceeded its bound"), + )) + except BaseException as error: + failures.append(_IntegrityFailure("linux-helper-stderr-read", error)) + try: + os.close(descriptor) + except BaseException as error: + failures.append(_IntegrityFailure("linux-process-fd-close:stderr", error)) + return bytes(data[:PROCESS_STARTUP_DIAGNOSTIC_MAX_BYTES]), failures + + def poll(self) -> int | None: + if self.returncode is not None: + return self.returncode + try: + waited_pid, status = os.waitpid(self.pid, os.WNOHANG) + except ChildProcessError as error: + if self.returncode is None: + raise TraceError("owned Linux child identity was lost before reap") from error + return self.returncode + if waited_pid == 0: + return None + self.returncode = os.waitstatus_to_exitcode(status) + return self.returncode + + def wait(self, timeout: float | None = None) -> int: + deadline = None if timeout is None else time.monotonic() + timeout + while self.poll() is None: + if deadline is not None and time.monotonic() >= deadline: + raise subprocess.TimeoutExpired(self.args, timeout) + time.sleep(0.01) + return int(self.returncode) + + def kill(self) -> None: + if self.poll() is None: + raise TraceError("owned Linux helper termination requires its stable pidfd") + + def communicate( + self, + input: bytes | None = None, + timeout: float | None = None, + ) -> tuple[bytes | None, bytes | None]: + if input is not None and self._stdin_fd is None: + raise ValueError("stdin is not a pipe") + output = bytearray() + errors = bytearray() + had_stdout = self._stdout_fd is not None + had_stderr = self._stderr_fd is not None + selector = selectors.DefaultSelector() + streams = {} + input_view = memoryview(input or b"") + input_offset = 0 + if self._stdin_fd is not None: + if input is None: + os.close(self._stdin_fd) + self._stdin_fd = None + else: + os.set_blocking(self._stdin_fd, False) + selector.register(self._stdin_fd, selectors.EVENT_WRITE) + for fd, buffer in ((self._stdout_fd, output), (self._stderr_fd, errors)): + if fd is not None: + os.set_blocking(fd, False) + selector.register(fd, selectors.EVENT_READ) + streams[fd] = buffer + deadline = None if timeout is None else time.monotonic() + timeout + try: + while selector.get_map() or self.poll() is None: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired( + self.args, + timeout, + output=bytes(output) if self._stdout_fd is not None else None, + stderr=bytes(errors) if self._stderr_fd is not None else None, + ) + else: + remaining = 0.05 + for key, events in selector.select(min(remaining, 0.05)): + if key.fd == self._stdin_fd and events & selectors.EVENT_WRITE: + try: + written = os.write( + key.fd, input_view[input_offset:input_offset + 65536]) + input_offset += written + except BrokenPipeError: + input_offset = len(input_view) + if input_offset == len(input_view): + selector.unregister(key.fd) + os.close(key.fd) + self._stdin_fd = None + continue + chunk = os.read(key.fd, 65536) + if chunk: + streams[key.fd].extend(chunk) + else: + selector.unregister(key.fd) + os.close(key.fd) + if key.fd == self._stdout_fd: + self._stdout_fd = None + if key.fd == self._stderr_fd: + self._stderr_fd = None + if not selector.get_map() and self.poll() is None: + time.sleep(0.01) + self.wait(timeout=0) + self._receive_protocol(b"COMPLETE") + self.completion_proven = True + finally: + selector.close() + self._stdout_fd = None + self._stderr_fd = None + return ( + bytes(output) if had_stdout else None, + bytes(errors) if had_stderr else None, + ) + + +def _linux_child_file_descriptors( + mode: Any, + target_fd: int, +) -> tuple[int | None, int | None]: + if mode is None: + return None, None + if mode == subprocess.PIPE: + read_fd, write_fd = os.pipe() + if target_fd == 0: + return write_fd, read_fd + return read_fd, write_fd + if mode == subprocess.DEVNULL: + flags = os.O_RDONLY if target_fd == 0 else os.O_WRONLY + descriptor = os.open(os.devnull, flags) + return None, descriptor + if mode == subprocess.STDOUT and target_fd == 2: + return None, subprocess.STDOUT + raise TraceError("Linux native helper containment received unsupported stream controls") + + +def _all_catchable_signals() -> set[int]: + return { + int(member) for member in signal.valid_signals() + if int(member) not in {signal.SIGKILL, signal.SIGSTOP} + } + + +def _require_zero_supplementary_groups() -> None: + try: + groups = os.getgroups() + except OSError as error: + raise TraceError( + f"cannot query Linux containment launcher supplementary groups: {error}") from error + if groups: + raise TraceError( + f"Linux containment launcher requires zero supplementary groups; found {len(groups)}") + + +def _start_linux_native_helper_process( + command: list[str], + launch: dict[str, Any], +) -> _LinuxNativeHelperProcess: + controls = dict(launch) + helper_path = str(controls.pop("_containment_helper_path")) + helper_descriptor = int(controls.pop("_containment_helper_descriptor")) + target_executable = str(controls.pop("executable", command[0])) + pass_fds = tuple(int(fd) for fd in controls.pop("pass_fds", ())) + environment = controls.pop("env", None) + working_directory = controls.pop("cwd", None) + if working_directory is not None: + raise TraceError("Linux native containment helper does not support cwd") + unknown_controls = set(controls) - {"stderr", "stdin", "stdout"} + if unknown_controls: + raise TraceError( + f"Linux native containment helper received unsupported controls: {sorted(unknown_controls)}") + parent_socket = None + child_socket = None + inheritable_before = {} + stdin_parent = None + stdin_child = None + stdout_parent = None + stdout_child = None + stderr_parent = None + stderr_child = None + process = None + primary_error = None + integrity_failures = [] + try: + stdin_parent, stdin_child = _linux_child_file_descriptors(controls.pop("stdin", None), 0) + stdout_parent, stdout_child = _linux_child_file_descriptors(controls.pop("stdout", None), 1) + stderr_parent, stderr_child = _linux_child_file_descriptors(controls.pop("stderr", None), 2) + parent_socket, child_socket = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET) + parent_socket.settimeout(PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS) + child_descriptors = [ + descriptor for descriptor in (stdin_child, stdout_child, stderr_child) + if descriptor is not None and descriptor != subprocess.STDOUT] + if any(descriptor <= 2 for descriptor in child_descriptors): + raise TraceError("Linux native containment helper requires intact standard descriptors") + inherited = {child_socket.fileno(), *pass_fds} + for descriptor in inherited: + inheritable_before[descriptor] = os.get_inheritable(descriptor) + os.set_inheritable(descriptor, True) + helper_argv = [ + helper_path, + "--protocol-fd", str(child_socket.fileno()), + "--expected-parent", str(os.getpid()), + "--exec-path", target_executable, + ] + for descriptor in pass_fds: + helper_argv.extend(("--keep-fd", str(descriptor))) + helper_argv.append("--") + helper_argv.extend(command) + file_actions = [] + for child_fd, target_fd in ( + (stdin_child, 0), (stdout_child, 1), (stderr_child, 2)): + if child_fd == subprocess.STDOUT: + file_actions.append((os.POSIX_SPAWN_DUP2, 1, 2)) + elif child_fd is not None: + file_actions.append((os.POSIX_SPAWN_DUP2, child_fd, target_fd)) + helper_exec_path = f"/proc/self/fd/{helper_descriptor}" + process_id = os.posix_spawn( + helper_exec_path, + helper_argv, + os.environ if environment is None else environment, + file_actions=file_actions, + setsid=True, + setsigmask=_all_catchable_signals(), + setsigdef=_all_catchable_signals(), + ) + process = _LinuxNativeHelperProcess( + command, + process_id, + protocol_socket=parent_socket, + stdin_fd=stdin_parent, + stdout_fd=stdout_parent, + stderr_fd=stderr_parent, + ) + parent_socket = None + stdin_parent = None + stdout_parent = None + stderr_parent = None + process.root_pidfd = _linux_open_pidfd(process_id) + except BaseException as error: + primary_error = error + for descriptor, previous in inheritable_before.items(): + try: + os.set_inheritable(descriptor, previous) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + "linux-launch-descriptor-inheritability-restore", error)) + if child_socket is not None: + try: + child_socket.close() + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + "linux-helper-child-protocol-close", error)) + for component, descriptor in ( + ("stdin", stdin_child), + ("stdout", stdout_child), + ("stderr", stderr_child)): + if descriptor is not None and descriptor != subprocess.STDOUT: + try: + os.close(descriptor) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"linux-helper-child-{component}-close", error)) + if process is not None: + process.launch_primary_error = primary_error + process.launch_integrity_failures = integrity_failures + return process + if parent_socket is not None: + try: + parent_socket.close() + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + "linux-helper-parent-protocol-close", error)) + for component, descriptor in ( + ("stdin", stdin_parent), + ("stdout", stdout_parent), + ("stderr", stderr_parent)): + if descriptor is not None: + try: + os.close(descriptor) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"linux-helper-parent-{component}-close", error)) + if primary_error is None and not integrity_failures: + raise TraceError("Linux native helper launch did not return a process") + if primary_error is None: + primary_error = integrity_failures.pop(0).error + if integrity_failures: + raise ExecutionIntegrityError( + f"Linux helper launch primary failure " + f"[{type(primary_error).__name__}: {primary_error}]" + + (f"; secondary integrity failures: " + f"{_format_integrity_failures(integrity_failures)}" if integrity_failures else ""), + primary_error=primary_error, + secondary_errors=integrity_failures, + quiescence_proven=False, + ) from primary_error + raise primary_error + + +def _linux_task_ids() -> set[int]: + task_root = Path("/proc/self/task") + if not task_root.is_dir(): + raise TraceError("Linux native containment requires procfs task identities") + try: + return {int(task.name) for task in task_root.iterdir()} + except (OSError, ValueError) as error: + raise TraceError(f"cannot read Linux native containment task identities: {error}") from error + + +def _linux_open_pidfd(process_id: int) -> int: + opener = getattr(os, "pidfd_open", None) + sender = getattr(signal, "pidfd_send_signal", None) + if opener is None or sender is None: + raise TraceError("Linux native containment requires pidfd signaling") + try: + return int(opener(process_id, 0)) + except ProcessLookupError: + raise + except OSError as error: + raise TraceError(f"cannot open stable Linux process identity: {error}") from error + + +def _linux_require_pidfd_support() -> None: + if getattr(os, "pidfd_open", None) is None or getattr(signal, "pidfd_send_signal", None) is None: + raise TraceError("Linux native containment requires pidfd signaling") + + +def _linux_signal_pidfd(pidfd: int, requested_signal: int) -> None: + sender = getattr(signal, "pidfd_send_signal", None) + if sender is None: + raise TraceError("Linux native containment requires pidfd signaling") + sender(pidfd, requested_signal) + + +def _linux_pidfd_has_exited(pidfd: int) -> bool: + poller = select.poll() + poller.register(pidfd, select.POLLIN) + return bool(poller.poll(0)) + + +def _linux_signal_owned_children( + containment: _ProcessContainment, + requested_signal: int, +) -> list[_IntegrityFailure]: + failures = [] + for component, pidfd in ( + ("linux-namespace-termination", containment.linux_namespace_pidfd), + ("linux-helper-termination", containment.linux_root_pidfd)): + if pidfd is None: + continue + try: + _linux_signal_pidfd(pidfd, requested_signal) + except ProcessLookupError: + pass + except BaseException as error: + failures.append(_IntegrityFailure(component, error)) + return failures + + +def _start_linux_native_helper(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + global _LINUX_HELPER_POISONED + if not _LINUX_HELPER_LOCK.acquire(blocking=False): + raise TraceError("Linux native containment helper is already active") + if _LINUX_HELPER_POISONED: + _LINUX_HELPER_LOCK.release() + raise TraceError("Linux native containment helper supervisor is not reusable") + process = None + pidfd = None + exec_released = False + try: + _require_zero_supplementary_groups() + _linux_require_pidfd_support() + if len(_linux_task_ids()) != 1: + raise TraceError("Linux native containment requires a single-threaded Python supervisor") + process = _start_linux_native_helper_process(command, launch) + pidfd = process.root_pidfd + containment = _ProcessContainment( + process=process, + linux_root_pidfd=pidfd, + linux_lock_held=True, + ) + if process.launch_primary_error is not None or process.launch_integrity_failures: + launch_primary = process.launch_primary_error + launch_failures = list(process.launch_integrity_failures) + if launch_primary is None: + launch_primary = launch_failures.pop(0).error + raise ExecutionIntegrityError( + f"Linux helper launch primary failure " + f"[{type(launch_primary).__name__}: {launch_primary}]" + + (f"; secondary integrity failures: " + f"{_format_integrity_failures(launch_failures)}" if launch_failures else ""), + primary_error=launch_primary, + secondary_errors=launch_failures, + quiescence_proven=False, + ) from launch_primary + if pidfd is None: + raise TraceError("Linux native containment helper identity is unavailable") + process.release_exec() + if process.namespace_pidfd is None or _linux_pidfd_has_exited(process.namespace_pidfd): + raise TraceError("Linux target PID namespace authority is unavailable before exec") + containment.linux_namespace_pidfd = process.namespace_pidfd + exec_released = True + containment.linux_exec_released = True + return containment + except BaseException as primary_error: + failures = [] + startup_stderr = b"" + if isinstance(primary_error, ExecutionIntegrityError): + failures.extend(primary_error.secondary_errors) + primary_error = primary_error.primary_error or primary_error + if process is not None: + failed_containment = _ProcessContainment( + process=process, + linux_root_pidfd=pidfd, + linux_namespace_pidfd=process.namespace_pidfd, + linux_lock_held=True, + linux_exec_released=exec_released, + ) + if process.namespace_pidfd is not None or exec_released: + cleanup = _terminate_process_tree(failed_containment) + else: + cleanup = process.abort_blocked() + failures.extend(cleanup.failures) + if cleanup.quiescence_proven: + startup_stderr, stderr_failures = process.collect_startup_stderr() + failures.extend(stderr_failures) + close_failures = _close_process_containment( + failed_containment, + quiescence_proven=cleanup.quiescence_proven, + ) + failures.extend(close_failures) + else: + _LINUX_HELPER_LOCK.release() + if failures: + raise ExecutionIntegrityError( + f"Linux containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]" + + (f"; helper stderr {startup_stderr!r}" if startup_stderr else "") + + "; " + f"secondary integrity failures: {_format_integrity_failures(failures)}", + primary_error=primary_error, + secondary_errors=failures, + quiescence_proven=False, + ) from primary_error + if process is not None: + raise ExecutionIntegrityError( + f"Linux containment startup primary failure " + f"[{type(primary_error).__name__}: {primary_error}]" + + (f"; helper stderr {startup_stderr!r}" if startup_stderr else ""), + primary_error=primary_error, + secondary_errors=[], + quiescence_proven=False, + ) from primary_error + raise + + +def _start_contained_process(command: list[str], launch: dict[str, Any]) -> _ProcessContainment: + if sys.platform == "win32": + return _start_windows_job_process(command, launch) + if sys.platform == "linux": + return _start_linux_native_helper(command, launch) + if sys.platform == "darwin" and getattr(_TEST_PROCESS_GROUP_CONTAINMENT, "enabled", False): + launch["start_new_session"] = True + process = subprocess.Popen(command, **launch) + return _ProcessContainment(process=process, test_process_group_id=process.pid) + raise TraceError("proven process containment is unavailable on this platform") + + +def _test_process_group_exists(process_group_id: int) -> bool: + try: + os.killpg(process_group_id, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + + +def _windows_job_active_processes(job_handle: int) -> int: + from ctypes import wintypes + + class BasicAccountingInformation(ctypes.Structure): + _fields_ = [ + ("TotalUserTime", ctypes.c_int64), + ("TotalKernelTime", ctypes.c_int64), + ("ThisPeriodTotalUserTime", ctypes.c_int64), + ("ThisPeriodTotalKernelTime", ctypes.c_int64), + ("TotalPageFaultCount", wintypes.DWORD), + ("TotalProcesses", wintypes.DWORD), + ("ActiveProcesses", wintypes.DWORD), + ("TotalTerminatedProcesses", wintypes.DWORD), + ] + + accounting = BasicAccountingInformation() + kernel32 = _windows_kernel32() + if not kernel32.QueryInformationJobObject( + job_handle, 1, ctypes.byref(accounting), ctypes.sizeof(accounting), None): + raise _windows_error("cannot query process containment job") + return int(accounting.ActiveProcesses) + + +def _close_windows_process_handle(process: subprocess.Popen[bytes]) -> None: + process_handle = getattr(process, "_handle", None) + if process_handle is None: + return + close = getattr(process_handle, "Close", None) + if not callable(close): + raise TraceError("subprocess process handle does not expose owned Close()") + close() + process._handle = None + + +def _process_tree_is_quiescent(containment: _ProcessContainment) -> bool: + if containment.job_handle is not None: + return _windows_job_active_processes(containment.job_handle) == 0 + if containment.linux_lock_held: + helper_exited = containment.process.poll() is not None + if containment.linux_namespace_pidfd is None: + return helper_exited and not containment.linux_exec_released + return helper_exited and _linux_pidfd_has_exited(containment.linux_namespace_pidfd) + if containment.test_process_group_id is not None: + return not _test_process_group_exists(containment.test_process_group_id) + raise TraceError("process containment identity is missing") + + +def _wait_for_process_tree_quiescence(containment: _ProcessContainment, deadline: float) -> None: + while not _process_tree_is_quiescent(containment): + if time.monotonic() >= deadline: + raise TraceError("process tree did not become quiescent before the cleanup deadline") + time.sleep(0.01) + + +def _terminate_process_tree(containment: _ProcessContainment) -> _ContainmentCleanup: + failures = [] + deadline = time.monotonic() + PROCESS_TREE_CLEANUP_TIMEOUT_SECONDS + process = containment.process + if containment.job_handle is not None: + try: + if not _windows_kernel32().TerminateJobObject(containment.job_handle, 1): + raise _windows_error("cannot terminate process containment job") + except BaseException as error: + failures.append(_IntegrityFailure("windows-job-termination", error)) + try: + process.communicate(timeout=max(0.01, deadline - time.monotonic())) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-reap", error)) + elif containment.linux_lock_held: + failures.extend(_linux_signal_owned_children(containment, signal.SIGTERM)) + try: + process.communicate(timeout=PROCESS_TREE_TERM_GRACE_SECONDS) + except subprocess.TimeoutExpired: + failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) + try: + process.communicate(timeout=max(0.01, deadline - time.monotonic())) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + failures.extend(_linux_signal_owned_children(containment, signal.SIGKILL)) + elif containment.test_process_group_id is not None: + try: + if _test_process_group_exists(containment.test_process_group_id): + try: + os.killpg(containment.test_process_group_id, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + term_deadline = min(deadline, time.monotonic() + PROCESS_TREE_TERM_GRACE_SECONDS) + try: + _wait_for_process_tree_quiescence(containment, term_deadline) + except TraceError: + if _test_process_group_exists(containment.test_process_group_id): + try: + os.killpg(containment.test_process_group_id, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-termination", error)) + try: + process.communicate(timeout=max(0.01, deadline - time.monotonic())) + except BaseException as error: + failures.append(_IntegrityFailure("direct-child-reap", error)) + else: + failures.append(_IntegrityFailure( + "process-tree-termination", TraceError("process containment identity is missing"))) + try: + _wait_for_process_tree_quiescence(containment, deadline) + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-quiescence", error)) + if containment.linux_exec_released and isinstance( + process, _LinuxNativeHelperProcess) and not process.completion_proven: + failures.append(_IntegrityFailure( + "linux-helper-completion", + TraceError("Linux native helper did not prove containment teardown"), + )) + quiescence_proven = not failures + if quiescence_proven: + try: + quiescence_proven = _process_tree_is_quiescent(containment) + except BaseException as error: + failures.append(_IntegrityFailure("process-tree-quiescence", error)) + quiescence_proven = False + return _ContainmentCleanup(failures, quiescence_proven) + + +def _run_contained_process( + command: list[str], + *, + label: str, + timeout: float | None, + input_data: bytes | None, + launch: dict[str, Any], +) -> _ContainedRun: + containment = None + try: + containment = _start_contained_process(command, launch) + except ExecutionIntegrityError as error: + return _ContainedRun( + None, + error.primary_error or error, + list(error.secondary_errors), + None, + error.quiescence_proven, + True, + ) + except BaseException as error: + return _ContainedRun(None, error, [], None, False, False) + try: + stdout, stderr = containment.process.communicate(input=input_data, timeout=timeout) + result = subprocess.CompletedProcess( + command, containment.process.returncode, stdout, stderr) + except BaseException as error: + cleanup = _terminate_process_tree(containment) + return _ContainedRun( + None, error, cleanup.failures, containment, cleanup.quiescence_proven, True) + try: + if _process_tree_is_quiescent(containment): + return _ContainedRun(result, None, [], containment, True, True) + except BaseException as error: + failures = [_IntegrityFailure("process-tree-quiescence", error)] + else: + error = TraceError(f"{label} process tree remained active after direct child exit") + failures = [] + cleanup = _terminate_process_tree(containment) + failures.extend(cleanup.failures) + return _ContainedRun( + None, error, failures, containment, cleanup.quiescence_proven, True) + + +def _close_process_containment( + containment: _ProcessContainment | None, + *, + quiescence_proven: bool, +) -> list[_IntegrityFailure]: + global _LINUX_HELPER_POISONED + global _LINUX_POISONED_CONTAINMENT + if containment is None: + return [] + failures = [] + if containment.linux_lock_held and not quiescence_proven: + failures.append(_IntegrityFailure( + "linux-helper-teardown", + TraceError("Linux native helper cannot be released before full tree quiescence"))) + _LINUX_HELPER_POISONED = True + _LINUX_POISONED_CONTAINMENT = containment + return failures + if containment.job_handle is not None: + try: + if not _windows_kernel32().CloseHandle(containment.job_handle): + raise _windows_error("cannot close process containment job") + containment.job_handle = None + except BaseException as error: + failures.append(_IntegrityFailure("containment-handle-close", error)) + try: + _close_windows_process_handle(containment.process) + except BaseException as error: + failures.append(_IntegrityFailure("windows-process-handle-close", error)) + linux_failure_count = len(failures) + if containment.linux_root_pidfd is not None: + try: + os.close(containment.linux_root_pidfd) + containment.linux_root_pidfd = None + if isinstance(containment.process, _LinuxNativeHelperProcess): + containment.process.root_pidfd = None + except BaseException as error: + failures.append(_IntegrityFailure("linux-root-pidfd-close", error)) + if containment.linux_namespace_pidfd is not None: + try: + os.close(containment.linux_namespace_pidfd) + containment.linux_namespace_pidfd = None + except BaseException as error: + failures.append(_IntegrityFailure("linux-namespace-pidfd-close", error)) + if isinstance(containment.process, _LinuxNativeHelperProcess): + failures.extend(containment.process.close_streams()) + if containment.linux_lock_held: + if len(failures) != linux_failure_count: + _LINUX_HELPER_POISONED = True + _LINUX_POISONED_CONTAINMENT = containment + containment.linux_lock_held = False + _LINUX_HELPER_LOCK.release() + return failures + + +def _format_integrity_failures(failures: list[_IntegrityFailure]) -> str: + return "; ".join( + f"{failure.component} [{type(failure.error).__name__}: {failure.error}]" + for failure in failures) + + +def _raise_execution_integrity_failures( + *, + label: str, + primary_error: BaseException | None, + integrity_failures: list[_IntegrityFailure], + containment_started: bool, + quiescence_proven: bool, +) -> None: + if primary_error is not None and containment_started: + suffix = "" + if integrity_failures: + suffix = f"; secondary integrity failures: {_format_integrity_failures(integrity_failures)}" + raise ExecutionIntegrityError( + f"{label} primary failure [{type(primary_error).__name__}: {primary_error}]{suffix}", + primary_error=primary_error, + secondary_errors=integrity_failures, + quiescence_proven=quiescence_proven, + ) from primary_error + if primary_error is not None: + raise primary_error + if integrity_failures: + raise ExecutionIntegrityError( + f"{label} integrity failures: {_format_integrity_failures(integrity_failures)}", + primary_error=None, + secondary_errors=integrity_failures, + quiescence_proven=quiescence_proven, + ) from integrity_failures[0].error + + +def _decode_subprocess_stream( + stream: bytes | None, + *, + text: bool, + encoding: str | None, + errors: str | None, +) -> bytes | str | None: + if stream is None or not text: + return stream + return stream.decode(encoding or "utf-8", errors or "strict") + + +def run_approved_executable( + command: list[str], + *, + path: Path, + runtime_policy: dict[str, Any], + expected_path: str, + expected_sha256: str, + label: str, + retained_fds: tuple[int, ...] = (), + **kwargs: Any, +) -> tuple[subprocess.CompletedProcess[Any], ExecutableFileReceipt]: + if sys.platform not in {"linux", "darwin", "win32"}: + raise TraceError(f"{label} immutable execution is unsupported on this platform") + if not command or command[0] != str(path): + raise TraceError(f"{label} command path differs from external approval") + protected_controls = { + "creationflags", "executable", "pass_fds", "preexec_fn", "process_group", "start_new_session"} + if protected_controls & kwargs.keys(): + raise TraceError(f"{label} execution parameters may not override immutable launch controls") + if sys.platform == "win32" and retained_fds: + raise TraceError(f"{label} retained descriptors are unsupported on Windows") + if any(type(descriptor) is not int or descriptor <= 2 for descriptor in retained_fds) or ( + len(set(retained_fds)) != len(retained_fds)): + raise TraceError(f"{label} retained descriptors are invalid") + try: + for retained_descriptor in retained_fds: + os.fstat(retained_descriptor) + except OSError as error: + raise TraceError(f"{label} retained descriptor is invalid: {error}") from error + timeout = kwargs.pop("timeout", None) + check = bool(kwargs.pop("check", False)) + capture_output = bool(kwargs.pop("capture_output", False)) + text = bool(kwargs.pop("text", kwargs.pop("universal_newlines", False))) + encoding = kwargs.pop("encoding", None) + errors = kwargs.pop("errors", None) + input_value = kwargs.pop("input", None) + if encoding is not None or errors is not None: + text = True + if input_value is not None and "stdin" in kwargs: + raise TraceError(f"{label} execution input conflicts with stdin") + if capture_output and ("stdout" in kwargs or "stderr" in kwargs): + raise TraceError(f"{label} capture_output conflicts with stdout or stderr") + if capture_output: + kwargs["stdout"] = subprocess.PIPE + kwargs["stderr"] = subprocess.PIPE + if input_value is not None: + kwargs["stdin"] = subprocess.PIPE + if isinstance(input_value, str): + input_data = input_value.encode(encoding or "utf-8", errors or "strict") + elif input_value is None or isinstance(input_value, bytes): + input_data = input_value + else: + raise TraceError(f"{label} execution input is invalid") + identity, descriptor = _approved_file_identity( + path, + install_root=Path(runtime_policy["install_root"]), + expected_owner_uid=runtime_policy["install_owner_uid"], + expected_path=expected_path, + expected_sha256=expected_sha256, + label=label, + executable=True, + ) + containment_helper: tuple[ExecutableFileReceipt, int] | None = None + if sys.platform == "linux": + helper_policy = _validate_containment_helper_policy( + runtime_policy.get("containment_helper"), + install_root=runtime_policy["install_root"], + revision=runtime_policy["revision"], + label=label, + ) + containment_helper = _approved_file_identity( + Path(helper_policy["path"]), + install_root=Path(runtime_policy["install_root"]), + expected_owner_uid=runtime_policy["install_owner_uid"], + expected_path=helper_policy["path"], + expected_sha256=helper_policy["sha256"], + label=f"{label} containment helper", + executable=True, + ) + runtime_files: list[tuple[ExecutableFileReceipt, int]] = [] + containment = None + result = None + decoded_result = None + primary_error = None + integrity_failures: list[_IntegrityFailure] = [] + containment_started = False + quiescence_proven = False + try: + for component in runtime_policy["runtime_receipt"]["components"]: + runtime_path = Path(runtime_policy["install_root"]) / "lib" / component["filename"] + runtime_files.append(_approved_file_identity( + runtime_path, + install_root=Path(runtime_policy["install_root"]), + expected_owner_uid=runtime_policy["install_owner_uid"], + expected_path=str(runtime_path), + expected_sha256=component["sha256"], + label=f"{label} runtime component {component['component']}", + executable=False, + )) + verify_approved_executable_identity(path, identity, label=label) + if containment_helper is not None: + verify_approved_executable_identity( + Path(containment_helper[0].path), + containment_helper[0], + label=f"{label} containment helper", + ) + for runtime_identity, _runtime_descriptor in runtime_files: + verify_approved_executable_identity( + Path(runtime_identity.path), + runtime_identity, + label=f"{label} runtime component", + ) + retained_descriptors = ( + descriptor, + *(item[1] for item in runtime_files), + *retained_fds, + ) + launch = dict(kwargs) + if sys.platform != "win32": + launch["pass_fds"] = retained_descriptors + if sys.platform == "linux": + launch["executable"] = f"/proc/self/fd/{descriptor}" + launch["_containment_helper_path"] = containment_helper[0].path + launch["_containment_helper_descriptor"] = containment_helper[1] + contained = _run_contained_process( + command, + label=label, + timeout=timeout, + input_data=input_data, + launch=launch, + ) + containment = contained.containment + containment_started = contained.process_started + quiescence_proven = contained.quiescence_proven + result = contained.result + primary_error = contained.primary_error + integrity_failures.extend(contained.integrity_failures) + if result is not None and check and result.returncode != 0 and primary_error is None: + primary_error = subprocess.CalledProcessError( + result.returncode, + result.args, + output=result.stdout, + stderr=result.stderr, + ) + try: + descriptor_after = os.fstat(descriptor) + if ( + descriptor_after.st_dev, + descriptor_after.st_ino, + descriptor_after.st_size, + descriptor_after.st_mtime_ns, + descriptor_after.st_ctime_ns, + ) != ( + identity.device, + identity.inode, + identity.byte_count, + identity.modified_ns, + identity.changed_ns, + ): + raise TraceError(f"{label} descriptor identity changed during execution") + except BaseException as error: + integrity_failures.append(_IntegrityFailure("executable-descriptor", error)) + try: + verify_approved_executable_identity(path, identity, label=label) + except BaseException as error: + integrity_failures.append(_IntegrityFailure("executable-path-root", error)) + if containment_helper is not None: + helper_identity, helper_descriptor = containment_helper + try: + helper_after = os.fstat(helper_descriptor) + if ( + helper_after.st_dev, + helper_after.st_ino, + helper_after.st_size, + helper_after.st_mtime_ns, + helper_after.st_ctime_ns, + ) != ( + helper_identity.device, + helper_identity.inode, + helper_identity.byte_count, + helper_identity.modified_ns, + helper_identity.changed_ns, + ): + raise TraceError(f"{label} containment helper descriptor changed during execution") + except BaseException as error: + integrity_failures.append(_IntegrityFailure("containment-helper-descriptor", error)) + try: + verify_approved_executable_identity( + Path(helper_identity.path), + helper_identity, + label=f"{label} containment helper", + ) + except BaseException as error: + integrity_failures.append(_IntegrityFailure("containment-helper-path-root", error)) + for runtime_identity, runtime_descriptor in runtime_files: + try: + runtime_after = os.fstat(runtime_descriptor) + if ( + runtime_after.st_dev, + runtime_after.st_ino, + runtime_after.st_uid, + stat.S_IMODE(runtime_after.st_mode), + runtime_after.st_nlink, + runtime_after.st_size, + runtime_after.st_mtime_ns, + runtime_after.st_ctime_ns, + ) != ( + runtime_identity.device, + runtime_identity.inode, + runtime_identity.owner_uid, + runtime_identity.mode, + runtime_identity.link_count, + runtime_identity.byte_count, + runtime_identity.modified_ns, + runtime_identity.changed_ns, + ): + raise TraceError(f"{label} runtime component descriptor changed during execution") + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"runtime-descriptor:{runtime_identity.path}", error)) + try: + verify_approved_executable_identity( + Path(runtime_identity.path), + runtime_identity, + label=f"{label} runtime component", + ) + except BaseException as error: + integrity_failures.append(_IntegrityFailure( + f"runtime-path-root:{runtime_identity.path}", error)) + except BaseException as error: + if primary_error is None: + primary_error = error + else: + integrity_failures.append(_IntegrityFailure("launcher-orchestration", error)) + finally: + teardown_failures = [] + for _runtime_identity, runtime_descriptor in runtime_files: + try: + os.close(runtime_descriptor) + except BaseException as error: + teardown_failures.append(_IntegrityFailure("runtime-descriptor-close", error)) + if containment_helper is not None: + try: + os.close(containment_helper[1]) + except BaseException as error: + teardown_failures.append(_IntegrityFailure("containment-helper-descriptor-close", error)) + try: + os.close(descriptor) + except BaseException as error: + teardown_failures.append(_IntegrityFailure("executable-descriptor-close", error)) + containment_failures = _close_process_containment( + containment, + quiescence_proven=quiescence_proven, + ) + teardown_failures.extend(containment_failures) + integrity_failures.extend(teardown_failures) + if teardown_failures: + quiescence_proven = False + if result is not None and primary_error is None: + try: + stdout = _decode_subprocess_stream( + result.stdout, text=text, encoding=encoding, errors=errors) + stderr = _decode_subprocess_stream( + result.stderr, text=text, encoding=encoding, errors=errors) + decoded_result = subprocess.CompletedProcess( + result.args, result.returncode, stdout, stderr) + except BaseException as error: + primary_error = error + _raise_execution_integrity_failures( + label=label, + primary_error=primary_error, + integrity_failures=integrity_failures, + containment_started=containment_started, + quiescence_proven=quiescence_proven, + ) + if decoded_result is None: + raise TraceError(f"{label} execution did not return a result") + return decoded_result, identity + + +def install_trust_evidence( + executable: ExecutableFileReceipt, + runtime_files: list[ExecutableFileReceipt], + additional_files: tuple[ExecutableFileReceipt, ...] = (), +) -> dict[str, Any]: + files = [executable, *runtime_files, *additional_files] + if any(item.install_root != executable.install_root or item.owner_uid != executable.owner_uid for item in files): + raise TraceError("approved install trust evidence spans multiple roots or owners") + directories: dict[str, tuple[str, int, int, int, int]] = {} + for item in files: + for directory in item.path_chain: + existing = directories.get(directory[0]) + if existing is not None and existing != directory: + raise TraceError("approved install directory identity is inconsistent") + directories[directory[0]] = directory + return { + "format": "dsv41-install-trust", + "version": 1, + "install_root": executable.install_root, + "owner_uid": executable.owner_uid, + "execution_uid": _execution_uid(), + "directories": [ + { + "path": item[0], + "device": item[1], + "inode": item[2], + "owner_uid": item[3], + "mode": item[4], + "effective_write_access": False, + "acl_entries": False, + } + for item in sorted(directories.values()) + ], + "files": [ + { + "path": item.path, + "device": item.device, + "inode": item.inode, + "owner_uid": item.owner_uid, + "mode": item.mode, + "link_count": item.link_count, + "byte_count": item.byte_count, + "modified_ns": item.modified_ns, + "changed_ns": item.changed_ns, + "sha256": item.sha256, + "effective_write_access": False, + "acl_entries": False, + } + for item in sorted(files, key=lambda value: value.path) + ], + } + + +def install_trust_sha256(record: dict[str, Any]) -> str: + validate_install_trust_evidence(record) + return sha256_bytes(canonical_json(record).encode("ascii")) + + +def validate_install_trust_evidence( + record: object, + policy: dict[str, Any] | None = None, +) -> dict[str, Any]: + if not isinstance(record, dict): + raise TraceError("install trust evidence is invalid") + _require_exact_keys( + record, + {"format", "version", "install_root", "owner_uid", "execution_uid", "directories", "files"}, + "install trust evidence", + ) + if record["format"] != "dsv41-install-trust" or record["version"] != 1: + raise TraceError("install trust evidence version is invalid") + install_root = _approval_path(record["install_root"], "install trust root") + owner_uid = record["owner_uid"] + execution_uid = record["execution_uid"] + if type(owner_uid) is not int or owner_uid < 0 or type(execution_uid) is not int or ( + execution_uid <= 0) or execution_uid == owner_uid: + raise TraceError("install trust owner or execution identity is invalid") + directories = record["directories"] + files = record["files"] + if not isinstance(directories, list) or not directories or not isinstance(files, list) or not files: + raise TraceError("install trust evidence is incomplete") + previous_path = None + directory_paths = set() + for directory in directories: + if not isinstance(directory, dict): + raise TraceError("install trust directory evidence is invalid") + _require_exact_keys( + directory, + { + "path", "device", "inode", "owner_uid", "mode", + "effective_write_access", "acl_entries", + }, + "install trust directory evidence", + ) + path = _approval_path(directory["path"], "install trust directory") + if path in directory_paths or (previous_path is not None and path <= previous_path): + raise TraceError("install trust directories are duplicated or unsorted") + directory_paths.add(path) + previous_path = path + if any(type(directory[key]) is not int or directory[key] < 0 for key in ( + "device", "inode", "owner_uid", "mode")) or ( + directory["mode"] & 0o022) or directory["effective_write_access"] is not False or ( + directory["acl_entries"] is not False): + raise TraceError("install trust directory is mutable or malformed") + if directory["owner_uid"] not in {0, owner_uid}: + raise TraceError("install trust directory owner is not trusted") + if (path == install_root or PurePosixPath(install_root) in PurePosixPath(path).parents) and ( + directory["owner_uid"] != owner_uid): + raise TraceError("install trust tree owner differs from approval") + previous_path = None + file_paths = set() + for file_record in files: + if not isinstance(file_record, dict): + raise TraceError("install trust file evidence is invalid") + _require_exact_keys( + file_record, + { + "path", "device", "inode", "owner_uid", "mode", "link_count", "byte_count", + "modified_ns", "changed_ns", "sha256", "effective_write_access", "acl_entries", + }, + "install trust file evidence", + ) + path = _approval_path(file_record["path"], "install trust file") + if path in file_paths or (previous_path is not None and path <= previous_path): + raise TraceError("install trust files are duplicated or unsorted") + file_paths.add(path) + previous_path = path + if any(type(file_record[key]) is not int or file_record[key] < 0 for key in ( + "device", "inode", "owner_uid", "mode", "link_count", "byte_count", + "modified_ns", "changed_ns")) or file_record["owner_uid"] != owner_uid or ( + file_record["mode"] & 0o222) or file_record["link_count"] != 1 or ( + file_record["effective_write_access"] is not False) or file_record["acl_entries"] is not False or ( + re.fullmatch(r"[0-9a-f]{64}", file_record.get("sha256", "")) is None): + raise TraceError("install trust file is mutable or malformed") + if policy is not None: + if install_root != policy["install_root"] or owner_uid != policy["install_owner_uid"]: + raise TraceError("install trust root differs from external approval") + expected_files = { + policy["executable_path"]: policy["executable_sha256"], + **{ + f"{install_root}/lib/{component['filename']}": component["sha256"] + for component in policy["runtime_receipt"]["components"] + }, + } + if "containment_helper" in policy: + helper = _validate_containment_helper_policy( + policy["containment_helper"], + install_root=install_root, + revision=policy["revision"], + label="external approval", + ) + expected_files[helper["path"]] = helper["sha256"] + observed_files = {item["path"]: item["sha256"] for item in files} + if observed_files != expected_files: + raise TraceError("install trust files differ from external approval") + expected_directories = set() + for path in expected_files: + current = PurePosixPath(path).parent + while True: + expected_directories.add(str(current)) + if str(current) == "/": + break + current = current.parent + if directory_paths != expected_directories: + raise TraceError("install trust directory coverage is incomplete") + return record + + +def validate_runtime_build_evidence( + record: object, + policy: dict[str, Any], + *, + label: str, +) -> dict[str, Any]: + if not isinstance(record, dict): + raise TraceError(f"{label} runtime build evidence is invalid") + _require_exact_keys( + record, + { + "revision", + "path", + "sha256", + "runtime_profile", + "runtime_receipt_sha256", + "runtime_libraries", + "runtime_libraries_post", + }, + f"{label} runtime build evidence", + ) + receipt_sha256 = sha256_bytes(canonical_json(policy["runtime_receipt"]).encode("ascii")) + expected = { + "revision": policy["revision"], + "path": policy["executable_path"], + "sha256": policy["executable_sha256"], + "runtime_profile": policy["runtime_profile"], + "runtime_receipt_sha256": receipt_sha256, + } + for key, value in expected.items(): + if record.get(key) != value: + raise TraceError(f"{label} runtime build {key} differs from external approval") + libraries = record["runtime_libraries"] + if libraries != record["runtime_libraries_post"] or not isinstance(libraries, list): + raise TraceError(f"{label} loaded runtime closure changed during execution") + expected_libraries = [] + for component in policy["runtime_receipt"]["components"]: + name = component["component"] + expected_libraries.append({ + "component": name, + "filename": component["filename"], + "path": f"{policy['install_root']}/lib/{component['filename']}", + "sha256": component["sha256"], + "role": { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + }.get(name, f"runtime:{name}"), + "revision": component["revision"], + }) + expected_libraries.sort(key=lambda item: item["path"]) + if libraries != expected_libraries: + raise TraceError(f"{label} loaded runtime libraries differ from external approval") + return record + + +def runtime_build_evidence_sha256(record: object, policy: dict[str, Any], *, label: str) -> str: + validated = validate_runtime_build_evidence(record, policy, label=label) + return sha256_bytes(canonical_json(validated).encode("ascii")) + + +def reject_loader_overrides(environment: dict[str, str] | None = None) -> None: + values = os.environ if environment is None else environment + active = sorted(name for name in FORBIDDEN_LOADER_ENVIRONMENT if values.get(name)) + if active: + raise TraceError("production trace execution forbids loader overrides: " + ", ".join(active)) + + +def canonical_json(data: Any) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def strict_json_loads(data: str) -> Any: + def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result = {} + for key, value in pairs: + if key in result: + raise TraceError(f"duplicate JSON key: {key}") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise TraceError(f"invalid JSON constant: {value}") + + try: + return json.loads( + data, + object_pairs_hook=reject_duplicates, + parse_constant=reject_constant, + ) + except json.JSONDecodeError as error: + raise TraceError(f"invalid JSON: {error}") from error + + +def trusted_ssh_keygen_path() -> Path: + if sys.platform == "win32": + return Path(r"C:\Windows\System32\OpenSSH\ssh-keygen.exe") + if sys.platform in ("darwin", "linux"): + return Path("/usr/bin/ssh-keygen") + raise TraceError(f"unsupported platform for trace signature verification: {sys.platform}") + + +def _validate_ssh_keygen(path: Path) -> Path: + if not path.is_absolute() or path.is_symlink(): + raise TraceError("trusted ssh-keygen path must be an absolute non-symlink") + try: + record = path.stat(follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot inspect trusted ssh-keygen: {error}") from error + _immutable_path_chain( + path, + install_root=path.parent.parent, + expected_owner_uid=0, + label="trusted ssh-keygen", + ) + if not stat.S_ISREG(record.st_mode) or record.st_uid != 0 or record.st_nlink != 1 or ( + stat.S_IMODE(record.st_mode) & 0o022) or _path_is_writable_by_execution_identity(path) or ( + _has_access_control_entries(path)) or not (stat.S_IMODE(record.st_mode) & 0o111): + raise TraceError("trusted ssh-keygen is not an executable regular file") + try: + result = subprocess.run( + [str(path), "-Y", "verify"], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot probe trusted ssh-keygen: {error}") from error + diagnostic = (result.stdout + result.stderr).lower() + if result.returncode == 0 or any( + marker in diagnostic for marker in ("unknown option", "illegal option", "unknown operation")): + raise TraceError("trusted ssh-keygen lacks required -Y signature support") + return path + + +def _ssh_environment() -> dict[str, str]: + environment = os.environ.copy() + environment.pop("SSH_AUTH_SOCK", None) + environment.pop("SSH_AGENT_PID", None) + return environment + + +def _validate_principal(principal: str) -> None: + if not isinstance(principal, str) or re.fullmatch(r"[A-Za-z0-9._@+-]{1,128}", principal) is None: + raise TraceError("trace signer principal is invalid") + + +def _signer_policy( + trusted_signers: dict[str, dict[str, str]], + principal: str) -> dict[str, str]: + _validate_principal(principal) + policy = trusted_signers.get(principal) + if not isinstance(policy, dict) or set(policy) != { + "public_key", "lane", "runtime", "runtime_profile"}: + raise TraceError(f"trace signer principal is not approved: {principal}") + expected = { + CANDIDATE_LANE: ("llama.cpp", "sibling-lib"), + ORACLE_LANE: ("ds4", "apple-metal"), + }.get(policy.get("lane")) + if expected is None or (policy.get("runtime"), policy.get("runtime_profile")) != expected: + raise TraceError("trace signer policy is invalid") + _normalize_public_key(policy.get("public_key", "")) + return policy + + +def _normalize_public_key(public_key: str, *, allow_comment: bool = False) -> str: + fields = public_key.strip().split() + expected_fields = len(fields) >= 2 if allow_comment else len(fields) == 2 + if not expected_fields or fields[0] != "ssh-ed25519" or re.fullmatch( + r"[A-Za-z0-9+/]+={0,2}", fields[1]) is None: + raise TraceError("trace signer public key must be an OpenSSH Ed25519 key") + return " ".join(fields[:2]) + + +def _approval_path(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.startswith("/") or value.startswith("//") or ( + ".." in PurePosixPath(value).parts or str(PurePosixPath(value)) != value): + raise TraceError(f"{label} is not an absolute canonical path") + return value + + +def _approval_digest(kind: str, approval_id: str, policy: dict[str, Any]) -> str: + if re.fullmatch(r"[A-Za-z0-9._-]{1,128}", approval_id) is None: + raise TraceError(f"{kind} approval ID is invalid") + record = { + "format": EXECUTABLE_APPROVAL_FORMAT, + "version": EXECUTABLE_APPROVAL_VERSION, + "kind": kind, + "id": approval_id, + "policy": policy, + } + return sha256_bytes(canonical_json(record).encode("ascii")) + + +def _validate_containment_helper_policy( + record: object, + *, + install_root: str, + revision: str, + label: str, +) -> dict[str, Any]: + if not isinstance(record, dict): + raise TraceError(f"{label} containment helper receipt is missing") + _require_exact_keys( + record, + { + "format", + "version", + "revision", + "filename", + "sha256", + "launcher_policy", + "supplementary_groups", + }, + f"{label} containment helper receipt", + ) + if record["format"] != "dsv41-containment-helper" or record["version"] != 2 or ( + record["revision"] != revision) or ( + record["filename"] != "llama-deepseek-v41-containment-helper") or re.fullmatch( + r"[0-9a-f]{64}", record.get("sha256", "")) is None or ( + record["launcher_policy"] != "zero-supplementary-groups-v1") or ( + record["supplementary_groups"] != []): + raise TraceError(f"{label} containment helper receipt is invalid") + result = dict(record) + result["path"] = f"{install_root}/bin/{record['filename']}" + return result + + +def approved_containment_helper_identity( + policy: dict[str, Any], + *, + label: str, +) -> ExecutableFileReceipt: + helper = _validate_containment_helper_policy( + policy.get("containment_helper"), + install_root=policy["install_root"], + revision=policy["revision"], + label=label, + ) + return approved_executable_identity( + Path(helper["path"]), + install_root=policy["install_root"], + expected_owner_uid=policy["install_owner_uid"], + expected_path=helper["path"], + expected_sha256=helper["sha256"], + label=f"{label} containment helper", + ) + + +def candidate_exporter_approval( + approval_id: str, + *, + policies: dict[str, dict[str, Any]] = APPROVED_CANDIDATE_EXPORTERS, +) -> tuple[dict[str, Any], str]: + policy = policies.get(approval_id) + if not isinstance(policy, dict): + raise TraceError(f"candidate exporter approval is not trusted: {approval_id}") + _require_exact_keys( + policy, + { + "runtime", + "repository", + "revision", + "base_revision", + "diff_sha256", + "install_root", + "install_owner_uid", + "executable_path", + "executable_sha256", + "containment_helper", + "runtime_profile", + "runtime_receipt", + }, + "candidate exporter approval", + ) + if policy["runtime"] != "llama.cpp" or policy["repository"] != REPOSITORY: + raise TraceError("candidate exporter approval runtime identity is invalid") + for key in ("revision", "base_revision"): + if re.fullmatch(r"[0-9a-f]{40}", policy.get(key, "")) is None: + raise TraceError(f"candidate exporter approval {key} is invalid") + for key in ("diff_sha256", "executable_sha256"): + if re.fullmatch(r"[0-9a-f]{64}", policy.get(key, "")) is None: + raise TraceError(f"candidate exporter approval {key} is invalid") + install_root = _approval_path(policy["install_root"], "candidate exporter approval install root") + if type(policy["install_owner_uid"]) is not int or policy["install_owner_uid"] < 0: + raise TraceError("candidate exporter approval install owner is invalid") + executable_path = _approval_path( + policy["executable_path"], "candidate exporter approval executable path") + if executable_path != f"{install_root}/bin/llama-deepseek-v41-trace": + raise TraceError("candidate exporter approval executable path is outside its install policy") + _validate_containment_helper_policy( + policy["containment_helper"], + install_root=install_root, + revision=policy["revision"], + label="candidate exporter approval", + ) + profile = policy["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("candidate exporter approval runtime profile is invalid") + _require_exact_keys( + profile, {"name", "components", "selected_backend_component"}, + "candidate exporter approval runtime profile") + if profile["name"] != "sibling-lib" or profile["selected_backend_component"] != "ggml-hip": + raise TraceError("candidate exporter approval runtime profile is invalid") + components = profile["components"] + if not isinstance(components, list) or components != sorted(components) or ( + len(components) != len(set(components))) or any( + not isinstance(component, str) or re.fullmatch(r"[a-z0-9-]+", component) is None + for component in components): + raise TraceError("candidate exporter approval components are invalid") + if not {"llama-common", "llama", "ggml", "ggml-base", "ggml-hip"}.issubset(set(components)): + raise TraceError("candidate exporter approval is missing required components") + receipt = policy["runtime_receipt"] + if not isinstance(receipt, dict): + raise TraceError("candidate exporter approval runtime receipt is invalid") + _require_exact_keys( + receipt, {"format", "version", "revision", "profile", "components"}, + "candidate exporter approval runtime receipt") + if receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or ( + receipt["revision"] != policy["revision"]) or receipt["profile"] != profile["name"]: + raise TraceError("candidate exporter approval runtime receipt identity is invalid") + receipt_components = receipt["components"] + if not isinstance(receipt_components, list) or receipt_components != sorted( + receipt_components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + raise TraceError("candidate exporter approval runtime receipt components are not canonical") + seen_components = set() + seen_filenames = set() + seen_digests = set() + for component in receipt_components: + if not isinstance(component, dict): + raise TraceError("candidate exporter approval runtime receipt component is invalid") + _require_exact_keys( + component, {"component", "filename", "sha256", "revision"}, + "candidate exporter approval runtime receipt component") + name = component["component"] + filename = component["filename"] + digest = component["sha256"] + revision = component["revision"] + if name not in components or name in seen_components: + raise TraceError("candidate exporter approval runtime receipt component name is invalid") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None or ( + filename in seen_filenames): + raise TraceError("candidate exporter approval runtime receipt filename is invalid") + if re.fullmatch(r"[0-9a-f]{64}", digest or "") is None or digest in seen_digests: + raise TraceError("candidate exporter approval runtime receipt digest is invalid") + revision_bearing = name in {"llama-common", "ggml-base"} + if (revision_bearing and revision != policy["revision"]) or ( + not revision_bearing and revision is not None): + raise TraceError("candidate exporter approval runtime receipt revision is invalid") + seen_components.add(name) + seen_filenames.add(filename) + seen_digests.add(digest) + if seen_components != set(components): + raise TraceError("candidate exporter approval receipt differs from its runtime profile") + return policy, _approval_digest("candidate-exporter", approval_id, policy) + + +def ds4_exporter_approval( + approval_id: str, + *, + policies: dict[str, dict[str, Any]] = APPROVED_DS4_EXPORTERS, +) -> tuple[dict[str, Any], str]: + policy = policies.get(approval_id) + if not isinstance(policy, dict): + raise TraceError(f"ds4 exporter approval is not trusted: {approval_id}") + _require_exact_keys( + policy, + { + "runtime", + "repository", + "revision", + "install_root", + "install_owner_uid", + "executable_path", + "executable_sha256", + "runtime_profile", + "runtime_receipt", + }, + "ds4 exporter approval", + ) + if policy["runtime"] != "ds4" or policy["repository"] != DS4_REPOSITORY or ( + policy["revision"] != DS4_REVISION): + raise TraceError("ds4 exporter approval runtime identity is invalid") + if re.fullmatch(r"[0-9a-f]{64}", policy.get("executable_sha256", "")) is None: + raise TraceError("ds4 exporter approval executable SHA-256 is invalid") + install_root = _approval_path(policy["install_root"], "ds4 exporter approval install root") + if type(policy["install_owner_uid"]) is not int or policy["install_owner_uid"] < 0: + raise TraceError("ds4 exporter approval install owner is invalid") + executable_path = _approval_path( + policy["executable_path"], "ds4 exporter approval executable path") + executable = PurePosixPath(executable_path) + if executable.parent != PurePosixPath(install_root) / "bin": + raise TraceError("ds4 exporter approval executable path is outside its install policy") + profile = policy["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("ds4 exporter approval runtime profile is invalid") + _require_exact_keys( + profile, {"name", "components", "selected_backend_component"}, + "ds4 exporter approval runtime profile") + components = profile["components"] + selected_backend = profile["selected_backend_component"] + if profile["name"] not in {"co-located", "sibling-lib"} or not isinstance(components, list) or ( + components != sorted(components)) or len(components) != len(set(components)) or not components or any( + not isinstance(component, str) or re.fullmatch(r"[a-z0-9-]+", component) is None + for component in components) or not isinstance(selected_backend, str) or ( + selected_backend not in components): + raise TraceError("ds4 exporter approval runtime profile is invalid") + receipt = policy["runtime_receipt"] + if not isinstance(receipt, dict): + raise TraceError("ds4 exporter approval runtime receipt is invalid") + _require_exact_keys( + receipt, {"format", "version", "revision", "profile", "components"}, + "ds4 exporter approval runtime receipt") + if receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or ( + receipt["revision"] != DS4_REVISION) or receipt["profile"] != profile["name"]: + raise TraceError("ds4 exporter approval runtime receipt identity is invalid") + receipt_components = receipt["components"] + if not isinstance(receipt_components, list) or receipt_components != sorted( + receipt_components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + raise TraceError("ds4 exporter approval runtime receipt components are not canonical") + seen_components = set() + seen_filenames = set() + seen_digests = set() + revision_bearing = 0 + for component in receipt_components: + if not isinstance(component, dict): + raise TraceError("ds4 exporter approval runtime receipt component is invalid") + _require_exact_keys( + component, {"component", "filename", "sha256", "revision"}, + "ds4 exporter approval runtime receipt component") + name = component["component"] + filename = component["filename"] + digest = component["sha256"] + revision = component["revision"] + if name not in components or name in seen_components: + raise TraceError("ds4 exporter approval runtime receipt component name is invalid") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None or ( + filename in seen_filenames): + raise TraceError("ds4 exporter approval runtime receipt filename is invalid") + if re.fullmatch(r"[0-9a-f]{64}", digest or "") is None or digest in seen_digests: + raise TraceError("ds4 exporter approval runtime receipt digest is invalid") + if revision not in {None, DS4_REVISION}: + raise TraceError("ds4 exporter approval runtime receipt revision is invalid") + revision_bearing += revision == DS4_REVISION + seen_components.add(name) + seen_filenames.add(filename) + seen_digests.add(digest) + if seen_components != set(components) or revision_bearing == 0: + raise TraceError("ds4 exporter approval receipt differs from its runtime profile") + return policy, _approval_digest("ds4-exporter", approval_id, policy) + + +def validate_tokenizer_policy(record: object) -> dict[str, bool]: + if not isinstance(record, dict): + raise TraceError("tokenizer policy is missing") + _require_exact_keys( + record, + { + "add_bos", + "parse_special", + "detokenize_special", + "remove_leading_bos_before_detokenize", + "require_round_trip", + }, + "tokenizer policy", + ) + if any(type(value) is not bool for value in record.values()): + raise TraceError("tokenizer policy values must be explicit booleans") + if record["parse_special"] is not True or record["detokenize_special"] is not True or ( + record["require_round_trip"] is not True) or ( + record["remove_leading_bos_before_detokenize"] != record["add_bos"]): + raise TraceError("tokenizer policy is not the exact prompt construction policy") + return dict(record) + + +def tokenizer_policy_sha256(record: object) -> str: + return sha256_bytes(canonical_json(validate_tokenizer_policy(record)).encode("ascii")) + + +def prompt_builder_approval( + approval_id: str, + *, + policies: dict[str, dict[str, Any]] = APPROVED_PROMPT_BUILDERS, +) -> tuple[dict[str, Any], str]: + policy = policies.get(approval_id) + if not isinstance(policy, dict): + raise TraceError(f"prompt builder approval is not trusted: {approval_id}") + _require_exact_keys( + policy, + { + "runtime", + "runtime_profile", + "repository", + "revision", + "install_root", + "install_owner_uid", + "executable_path", + "executable_sha256", + "containment_helper", + "source_root", + "runtime_receipt", + "model_sha256", + "corpora", + "tokenizer", + "prompts", + }, + "prompt builder approval", + ) + if policy["runtime"] != "llama.cpp" or ( + policy["repository"] != REPOSITORY) or re.fullmatch( + r"[0-9a-f]{40}", policy.get("revision", "")) is None: + raise TraceError("prompt builder approval runtime identity is invalid") + if policy["model_sha256"] != MODEL_SHA256 or re.fullmatch( + r"[0-9a-f]{64}", policy.get("executable_sha256", "")) is None: + raise TraceError("prompt builder approval executable or model identity is invalid") + install_root = _approval_path(policy["install_root"], "prompt builder approval install root") + if type(policy["install_owner_uid"]) is not int or policy["install_owner_uid"] < 0: + raise TraceError("prompt builder approval install owner is invalid") + executable_path = _approval_path( + policy["executable_path"], "prompt builder approval executable path") + source_root = _approval_path(policy["source_root"], "prompt builder approval source root") + if executable_path != f"{install_root}/bin/llama-deepseek-v41-prompt-builder": + raise TraceError("prompt builder approval executable path is outside its install policy") + _validate_containment_helper_policy( + policy["containment_helper"], + install_root=install_root, + revision=policy["revision"], + label="prompt builder approval", + ) + profile = policy["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("prompt builder approval runtime profile is invalid") + _require_exact_keys( + profile, {"name", "components", "selected_backend_component"}, + "prompt builder approval runtime profile") + if profile["name"] != "sibling-lib" or not isinstance(profile["components"], list) or ( + profile["components"] != sorted(profile["components"])) or len( + profile["components"]) != len(set(profile["components"])) or not { + "llama-common", "llama", "ggml", "ggml-base" + }.issubset(set(profile["components"])) or ( + profile["selected_backend_component"] != "ggml-hip") or ( + profile["selected_backend_component"] not in profile["components"]): + raise TraceError("prompt builder approval runtime profile is invalid") + receipt = policy["runtime_receipt"] + if not isinstance(receipt, dict): + raise TraceError("prompt builder approval runtime receipt is invalid") + _require_exact_keys( + receipt, {"format", "version", "revision", "profile", "components"}, + "prompt builder approval runtime receipt") + if receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or ( + receipt["revision"] != policy["revision"]) or receipt["profile"] != profile["name"]: + raise TraceError("prompt builder approval runtime receipt identity is invalid") + receipt_components = receipt["components"] + if not isinstance(receipt_components, list) or receipt_components != sorted( + receipt_components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + raise TraceError("prompt builder approval runtime receipt components are not canonical") + seen_components = set() + seen_filenames = set() + seen_digests = set() + for component in receipt_components: + if not isinstance(component, dict): + raise TraceError("prompt builder approval runtime receipt component is invalid") + _require_exact_keys( + component, {"component", "filename", "sha256", "revision"}, + "prompt builder approval runtime receipt component") + name = component["component"] + filename = component["filename"] + digest = component["sha256"] + revision = component["revision"] + if name not in profile["components"] or name in seen_components: + raise TraceError("prompt builder approval runtime receipt component name is invalid") + if not isinstance(filename, str) or re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None or ( + filename in seen_filenames): + raise TraceError("prompt builder approval runtime receipt filename is invalid") + if re.fullmatch(r"[0-9a-f]{64}", digest or "") is None or digest in seen_digests: + raise TraceError("prompt builder approval runtime receipt digest is invalid") + revision_bearing = name in {"llama-common", "ggml-base"} + if (revision_bearing and revision != policy["revision"]) or ( + not revision_bearing and revision is not None): + raise TraceError("prompt builder approval runtime receipt revision is invalid") + seen_components.add(name) + seen_filenames.add(filename) + seen_digests.add(digest) + if seen_components != set(profile["components"]): + raise TraceError("prompt builder approval receipt differs from its runtime profile") + if policy["corpora"] != CORPUS_SHA256: + raise TraceError("prompt builder approval corpus policy is invalid") + validate_tokenizer_policy(policy["tokenizer"]) + prompts = policy["prompts"] + if not isinstance(prompts, list) or not prompts: + raise TraceError("prompt builder approval prompt policy is empty") + previous_key = None + seen_keys = set() + for prompt in prompts: + if not isinstance(prompt, dict): + raise TraceError("prompt builder approval prompt record is invalid") + _require_exact_keys( + prompt, + { + "corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", + "prompt_sha256", "prompt_byte_count", + }, + "prompt builder approval prompt record", + ) + corpus_name = prompt["corpus_name"] + if corpus_name not in CORPUS_SHA256 or prompt["corpus_sha256"] != CORPUS_SHA256[corpus_name]: + raise TraceError("prompt builder approval prompt corpus identity is invalid") + context = prompt["context"] + decode_steps = prompt["decode_steps"] + target_tokens = prompt["target_tokens"] + if type(context) is not int or context < 2 or type(decode_steps) is not int or decode_steps < 1 or ( + target_tokens != context - decode_steps): + raise TraceError("prompt builder approval prompt configuration is invalid") + if re.fullmatch(r"[0-9a-f]{64}", prompt.get("prompt_sha256", "")) is None or ( + type(prompt.get("prompt_byte_count")) is not int or prompt["prompt_byte_count"] <= 0): + raise TraceError("prompt builder approval prompt output identity is invalid") + key = (corpus_name, context, decode_steps) + if key in seen_keys or (previous_key is not None and key <= previous_key): + raise TraceError("prompt builder approval prompt records are duplicated or unsorted") + seen_keys.add(key) + previous_key = key + return policy, _approval_digest("prompt-builder", approval_id, policy) + + +def approved_prompt_record( + policy: dict[str, Any], + *, + corpus_name: str, + context: int, + decode_steps: int, +) -> dict[str, Any]: + matches = [ + prompt for prompt in policy["prompts"] + if prompt["corpus_name"] == corpus_name and + prompt["context"] == context and + prompt["decode_steps"] == decode_steps + ] + if len(matches) != 1: + raise TraceError("prompt builder approval does not contain the requested prompt configuration") + return matches[0] + + +def _read_external_regular_file( + path: Path, + label: str, + *, + trusted_root: Path, + expected_owner_uid: int, + test_only_trust: bool = False, +) -> bytes: + if not path.is_absolute() or str(path.resolve()) != str(path): + raise TraceError(f"{label} path must be absolute, canonical, and non-symlinked") + if path != trusted_root and trusted_root not in path.parents: + raise TraceError(f"{label} is outside its trusted root") + if not test_only_trust: + _immutable_path_chain( + path, + install_root=trusted_root, + expected_owner_uid=expected_owner_uid, + label=label, + ) + try: + before = path.stat(follow_symlinks=False) + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + opened = os.fstat(descriptor) + except OSError as error: + raise TraceError(f"cannot inspect {label}: {error}") from error + identity = ( + before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) + if identity != ( + opened.st_dev, opened.st_ino, opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) or ( + not stat.S_ISREG(opened.st_mode)) or opened.st_nlink != 1 or ( + not test_only_trust and ( + opened.st_uid != expected_owner_uid or stat.S_IMODE(opened.st_mode) & 0o222 or + _path_is_writable_by_execution_identity(path) or _has_access_control_entries(path))): + os.close(descriptor) + raise TraceError(f"{label} must be an immutable trusted-owned one-link regular file") + try: + with os.fdopen(os.dup(descriptor), "rb") as stream: + data = stream.read() + after = os.fstat(descriptor) + path_after = path.stat(follow_symlinks=False) + except OSError as error: + os.close(descriptor) + raise TraceError(f"cannot read {label}: {error}") from error + if identity != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) or ( + identity != ( + path_after.st_dev, path_after.st_ino, path_after.st_size, + path_after.st_mtime_ns, path_after.st_ctime_ns)) or len(data) != before.st_size: + os.close(descriptor) + raise TraceError(f"{label} changed while reading") + os.close(descriptor) + return data + + +def load_executable_approval_policy( + policy_path: Path, + signature_path: Path, + *, + expected_principal: str, + trusted_approvers: dict[str, dict[str, Any]] = APPROVED_EXECUTABLE_APPROVERS, + ssh_keygen: Path | None = None, + forbidden_roots: Iterable[Path] = (), + test_only_trust: bool = False, +) -> ExecutableApprovalPolicy: + _validate_principal(expected_principal) + resolved_policy = policy_path.resolve() + resolved_signature = signature_path.resolve() + for forbidden_root in forbidden_roots: + root = forbidden_root.resolve() + if resolved_policy == root or root in resolved_policy.parents or ( + resolved_signature == root) or root in resolved_signature.parents: + raise TraceError("executable approval policy and signature must be outside protected output roots") + approver = trusted_approvers.get(expected_principal) + if not isinstance(approver, dict) or set(approver) != { + "public_key", "policy_root", "owner_uid"}: + raise TraceError(f"executable approval principal is not trusted: {expected_principal}") + policy_root = Path(_approval_path(approver["policy_root"], "executable approval trusted root")) + owner_uid = approver["owner_uid"] + if type(owner_uid) is not int or owner_uid < 0: + raise TraceError("executable approval trusted owner is invalid") + approved_key = _normalize_public_key(approver["public_key"]) + policy_bytes = _read_external_regular_file( + policy_path, + "executable approval policy", + trusted_root=policy_root, + expected_owner_uid=owner_uid, + test_only_trust=test_only_trust, + ) + signature_bytes = _read_external_regular_file( + signature_path, + "executable approval signature", + trusted_root=policy_root, + expected_owner_uid=owner_uid, + test_only_trust=test_only_trust, + ) + try: + policy = strict_json_loads(policy_bytes.decode("ascii")) + signature = signature_bytes.decode("ascii") + except UnicodeError as error: + raise TraceError("executable approval policy and signature must be ASCII") from error + if policy_bytes != _canonical_json_bytes(policy): + raise TraceError("executable approval policy is not canonical") + if not isinstance(policy, dict): + raise TraceError("executable approval policy must be a JSON object") + _require_exact_keys( + policy, + { + "format", "version", "principal", "verifier_repository", "verifier_revision", + "candidate_exporters", "ds4_exporters", "prompt_builders", + }, + "executable approval policy", + ) + if policy["format"] != EXECUTABLE_APPROVAL_FORMAT or ( + policy["version"] != EXECUTABLE_APPROVAL_VERSION) or ( + policy["principal"] != expected_principal) or ( + policy["verifier_repository"] != REPOSITORY) or re.fullmatch( + r"[0-9a-f]{40}", policy.get("verifier_revision", "")) is None: + raise TraceError("executable approval policy identity is invalid") + candidate_exporters = policy["candidate_exporters"] + ds4_exporters = policy["ds4_exporters"] + prompt_builders = policy["prompt_builders"] + if not isinstance(candidate_exporters, dict) or not isinstance(ds4_exporters, dict) or ( + not isinstance(prompt_builders, dict)): + raise TraceError("executable approval policy maps are invalid") + for approval_id in sorted(candidate_exporters): + candidate_exporter_approval(approval_id, policies=candidate_exporters) + for approval_id in sorted(ds4_exporters): + ds4_policy, _digest = ds4_exporter_approval(approval_id, policies=ds4_exporters) + if ds4_policy["revision"] == policy["verifier_revision"]: + raise TraceError("ds4 exporter producer revision must differ from the verifier revision") + for approval_id in sorted(prompt_builders): + prompt_builder_approval(approval_id, policies=prompt_builders) + if not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or not signature.endswith( + "-----END SSH SIGNATURE-----\n"): + raise TraceError("executable approval signature is invalid") + executable = _validate_ssh_keygen(ssh_keygen or trusted_ssh_keygen_path()) + with tempfile.TemporaryDirectory(prefix="dsv41-approval-verify-") as temp: + temporary = Path(temp) + allowed_signers = temporary / "allowed_signers" + signature_file = temporary / "signature" + allowed_signers.write_text(f"{expected_principal} {approved_key}\n", encoding="ascii") + signature_file.write_text(signature, encoding="ascii") + try: + result = subprocess.run( + [ + str(executable), + "-Y", "verify", + "-f", str(allowed_signers), + "-I", expected_principal, + "-n", EXECUTABLE_APPROVAL_NAMESPACE, + "-s", str(signature_file), + ], + input=policy_bytes, + check=False, + capture_output=True, + timeout=30, + env=_ssh_environment(), + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot verify executable approval signature: {error}") from error + if result.returncode != 0: + raise TraceError("executable approval signature verification failed") + return ExecutableApprovalPolicy( + principal=expected_principal, + verifier_revision=policy["verifier_revision"], + candidate_exporters=candidate_exporters, + ds4_exporters=ds4_exporters, + prompt_builders=prompt_builders, + sha256=sha256_bytes(policy_bytes), + ) + + +def approval_binding( + kind: str, + approval_id: str, + digest: str, + trust_sha256: str, +) -> dict[str, str]: + if kind not in {"candidate_exporter", "ds4_exporter", "prompt_builder"} or re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", approval_id) is None or re.fullmatch( + r"[0-9a-f]{64}", digest) is None or re.fullmatch( + r"[0-9a-f]{64}", trust_sha256) is None: + raise TraceError("execution approval binding is invalid") + return {"id": approval_id, "sha256": digest, "install_trust_sha256": trust_sha256} + + +def validate_execution_authorization( + manifest: dict[str, Any], + *, + policy: dict[str, str], + expected_lane: str, + expected_challenge: str, + expected_run_id: str, + verification_unix: int, + candidate_exporter_policies: dict[str, dict[str, Any]], + ds4_exporter_policies: dict[str, dict[str, Any]], + prompt_builder_policies: dict[str, dict[str, Any]], + expected_candidate_exporter_policy_id: str | None, + expected_ds4_exporter_policy_id: str | None, + expected_prompt_builder_policy_id: str, + expected_approval_policy_sha256: str, + expected_verifier_revision: str, + seen_run_ids: set[str] | None = None) -> None: + if expected_lane not in {CANDIDATE_LANE, ORACLE_LANE}: + raise TraceError("externally expected execution lane is invalid") + if re.fullmatch(r"[0-9a-f]{64}", expected_challenge) is None: + raise TraceError("externally expected execution challenge is invalid") + run_prefix = "strix-llama-" if expected_lane == CANDIDATE_LANE else "apple-ds4-" + if re.fullmatch(re.escape(run_prefix) + r"[A-Za-z0-9._-]{1,96}", expected_run_id) is None: + raise TraceError("externally expected lane run ID is invalid") + if type(verification_unix) is not int or verification_unix <= 0: + raise TraceError("trace verification time is invalid") + authorization = manifest.get("authorization") + if not isinstance(authorization, dict): + raise TraceError("manifest execution authorization is missing") + _require_exact_keys( + authorization, + { + "format", "version", "lane", "challenge", "run_id", "issued_unix", + "expires_unix", "approval_policy_sha256", "verifier_revision", + "tokenizer_policy_sha256", "approvals", + }, + "manifest execution authorization", + ) + if authorization.get("format") != AUTHORIZATION_FORMAT or ( + authorization.get("version") != AUTHORIZATION_VERSION): + raise TraceError("manifest execution authorization version is invalid") + if authorization.get("lane") != expected_lane or policy["lane"] != expected_lane: + raise TraceError("trace signer is not approved for the expected execution lane") + if authorization.get("challenge") != expected_challenge: + raise TraceError("manifest execution challenge differs from the external challenge") + if authorization.get("run_id") != expected_run_id: + raise TraceError("manifest lane run ID differs from the external run ID") + if re.fullmatch(r"[0-9a-f]{64}", expected_approval_policy_sha256) is None or ( + authorization.get("approval_policy_sha256") != expected_approval_policy_sha256): + raise TraceError("manifest executable approval policy differs from the external policy") + if re.fullmatch(r"[0-9a-f]{40}", expected_verifier_revision) is None or ( + authorization.get("verifier_revision") != expected_verifier_revision): + raise TraceError("manifest verifier revision differs from the external approval policy") + issued_unix = authorization.get("issued_unix") + expires_unix = authorization.get("expires_unix") + if type(issued_unix) is not int or type(expires_unix) is not int or ( + issued_unix <= 0 or expires_unix <= issued_unix or + expires_unix - issued_unix > MAX_AUTHORIZATION_LIFETIME_SECONDS): + raise TraceError("manifest execution authorization validity window is invalid") + if verification_unix < issued_unix or verification_unix > expires_unix: + raise TraceError("manifest execution authorization is expired or not yet valid") + runtime = manifest.get("runtime") + if runtime != policy["runtime"]: + raise TraceError("trace signer runtime role does not match the signed manifest") + runtime_profile = ( + manifest.get("build", {}).get("runtime_profile", {}).get("name") + if runtime == "llama.cpp" + else manifest.get("accelerator", {}).get("runtime_kind") + ) + if runtime_profile != policy["runtime_profile"]: + raise TraceError("trace signer runtime profile does not match the signed manifest") + _prompt_policy, prompt_digest = prompt_builder_approval( + expected_prompt_builder_policy_id, policies=prompt_builder_policies) + if authorization.get("tokenizer_policy_sha256") != tokenizer_policy_sha256(_prompt_policy["tokenizer"]): + raise TraceError("manifest tokenizer policy differs from external prompt approval") + approvals = authorization["approvals"] + required_approvals = {"prompt_builder"} + if expected_lane == CANDIDATE_LANE: + required_approvals.add("candidate_exporter") + else: + required_approvals.add("ds4_exporter") + if not isinstance(approvals, dict): + raise TraceError("manifest execution approval bindings are invalid") + _require_exact_keys(approvals, required_approvals, "manifest execution approval bindings") + prompt_binding = approvals["prompt_builder"] + if not isinstance(prompt_binding, dict) or set(prompt_binding) != { + "id", "sha256", "install_trust_sha256"} or prompt_binding.get("id") != ( + expected_prompt_builder_policy_id) or prompt_binding.get("sha256") != prompt_digest or re.fullmatch( + r"[0-9a-f]{64}", prompt_binding.get("install_trust_sha256", "")) is None: + raise TraceError("manifest prompt builder approval differs from external policy") + if expected_lane == CANDIDATE_LANE: + if expected_candidate_exporter_policy_id is None: + raise TraceError("external candidate exporter approval ID is required") + if expected_ds4_exporter_policy_id is not None: + raise TraceError("candidate verification must not specify a ds4 exporter approval") + _candidate_policy, candidate_digest = candidate_exporter_approval( + expected_candidate_exporter_policy_id, policies=candidate_exporter_policies) + candidate_binding = approvals["candidate_exporter"] + if not isinstance(candidate_binding, dict) or set(candidate_binding) != { + "id", "sha256", "install_trust_sha256"} or candidate_binding.get("id") != ( + expected_candidate_exporter_policy_id) or candidate_binding.get("sha256") != ( + candidate_digest) or re.fullmatch( + r"[0-9a-f]{64}", candidate_binding.get("install_trust_sha256", "")) is None: + raise TraceError("manifest candidate exporter approval differs from external policy") + else: + if expected_candidate_exporter_policy_id is not None: + raise TraceError("oracle verification must not specify a candidate exporter approval") + if expected_ds4_exporter_policy_id is None: + raise TraceError("external ds4 exporter approval ID is required") + _ds4_policy, ds4_digest = ds4_exporter_approval( + expected_ds4_exporter_policy_id, policies=ds4_exporter_policies) + ds4_binding = approvals["ds4_exporter"] + if not isinstance(ds4_binding, dict) or set(ds4_binding) != { + "id", "sha256", "install_trust_sha256"} or ds4_binding.get("id") != ( + expected_ds4_exporter_policy_id) or ds4_binding.get("sha256") != ( + ds4_digest) or re.fullmatch( + r"[0-9a-f]{64}", ds4_binding.get("install_trust_sha256", "")) is None: + raise TraceError("manifest ds4 exporter approval differs from external policy") + if seen_run_ids is not None: + if expected_run_id in seen_run_ids: + raise TraceError("trace lane run ID was reused") + seen_run_ids.add(expected_run_id) + + +def execution_authorization( + *, + lane: str, + challenge: str, + run_id: str, + issued_unix: int, + expires_unix: int, + approval_policy_sha256: str, + verifier_revision: str, + tokenizer_policy_sha256_value: str, + approvals: dict[str, dict[str, str]]) -> dict[str, Any]: + if lane not in {CANDIDATE_LANE, ORACLE_LANE}: + raise TraceError("execution authorization lane is invalid") + if re.fullmatch(r"[0-9a-f]{64}", challenge) is None: + raise TraceError("execution authorization challenge is invalid") + run_prefix = "strix-llama-" if lane == CANDIDATE_LANE else "apple-ds4-" + if re.fullmatch(re.escape(run_prefix) + r"[A-Za-z0-9._-]{1,96}", run_id) is None: + raise TraceError("execution authorization run ID is invalid") + if type(issued_unix) is not int or type(expires_unix) is not int or ( + issued_unix <= 0 or expires_unix <= issued_unix or + expires_unix - issued_unix > MAX_AUTHORIZATION_LIFETIME_SECONDS): + raise TraceError("execution authorization validity window is invalid") + if re.fullmatch(r"[0-9a-f]{64}", approval_policy_sha256) is None or re.fullmatch( + r"[0-9a-f]{40}", verifier_revision) is None: + raise TraceError("execution authorization approval policy identity is invalid") + if re.fullmatch(r"[0-9a-f]{64}", tokenizer_policy_sha256_value) is None: + raise TraceError("execution authorization tokenizer policy identity is invalid") + required_approvals = {"prompt_builder"} + if lane == CANDIDATE_LANE: + required_approvals.add("candidate_exporter") + else: + required_approvals.add("ds4_exporter") + if not isinstance(approvals, dict): + raise TraceError("execution authorization approvals are invalid") + _require_exact_keys(approvals, required_approvals, "execution authorization approvals") + for kind, binding in approvals.items(): + if not isinstance(binding, dict): + raise TraceError("execution authorization approval binding is invalid") + if binding != approval_binding( + kind, + binding.get("id", ""), + binding.get("sha256", ""), + binding.get("install_trust_sha256", ""), + ): + raise TraceError("execution authorization approval binding is invalid") + authorization = { + "format": AUTHORIZATION_FORMAT, + "version": AUTHORIZATION_VERSION, + "lane": lane, + "challenge": challenge, + "run_id": run_id, + "issued_unix": issued_unix, + "expires_unix": expires_unix, + "approval_policy_sha256": approval_policy_sha256, + "verifier_revision": verifier_revision, + "tokenizer_policy_sha256": tokenizer_policy_sha256_value, + "approvals": approvals, + } + return authorization + + +def bind_execution_authorization(root: Path, authorization: dict[str, Any]) -> None: + manifest_path = root / MANIFEST_NAME + try: + manifest = strict_json_loads(manifest_path.read_text(encoding="ascii")) + except (OSError, UnicodeError, TraceError) as error: + raise TraceError(f"cannot bind execution authorization: {error}") from error + if not isinstance(manifest, dict): + raise TraceError("cannot bind execution authorization to a non-object manifest") + if "authorization" in manifest: + raise TraceError("manifest execution authorization is already present") + manifest["authorization"] = authorization + temporary = manifest_path.with_suffix(".tmp") + temporary.write_bytes(_canonical_json_bytes(manifest)) + os.replace(temporary, manifest_path) + + +def validate_signing_identity( + private_key: Path, + principal: str, + *, + trusted_signers: dict[str, dict[str, str]] = APPROVED_TRACE_SIGNERS, + ssh_keygen: Path | None = None, + forbidden_root: Path | None = None) -> tuple[Path, str]: + policy = _signer_policy(trusted_signers, principal) + expected_key = policy["public_key"] + executable = _validate_ssh_keygen(ssh_keygen or trusted_ssh_keygen_path()) + key_path = private_key.resolve() + if private_key.is_symlink() or not key_path.is_file(): + raise TraceError("trace signing key must be a regular non-symlink file") + key_stat = key_path.stat() + if os.name != "nt": + if key_stat.st_uid != os.getuid(): + raise TraceError("trace signing key is not owned by the current user") + if key_stat.st_mode & 0o077: + raise TraceError("trace signing key permissions are too broad") + if forbidden_root is not None: + try: + key_path.relative_to(forbidden_root.resolve()) + except ValueError: + pass + else: + raise TraceError("trace signing key must be outside the bundle") + try: + result = subprocess.run( + [str(executable), "-y", "-f", str(key_path)], + check=False, + capture_output=True, + text=True, + timeout=10, + env=_ssh_environment(), + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot derive trace signing public key: {error}") from error + if result.returncode != 0: + raise TraceError("cannot derive trace signing public key") + derived_key = _normalize_public_key(result.stdout, allow_comment=True) + if derived_key != _normalize_public_key(expected_key): + raise TraceError("trace signing key does not match the approved signer") + return executable, derived_key + + +def _canonical_json_bytes(data: Any) -> bytes: + return (canonical_json(data) + "\n").encode("ascii") + + +def _bundle_path_parts(relative: str) -> tuple[str, ...]: + if not isinstance(relative, str) or not relative or any(ord(character) > 0x7f for character in relative): + raise TraceError(f"trace path is not portable ASCII: {relative!r}") + if "\\" in relative or "%" in relative or relative.startswith("/") or relative.startswith("//") or ( + re.match(r"^[A-Za-z]:", relative) is not None): + raise TraceError(f"trace path is outside the bundle: {relative}") + parts = relative.split("/") + if any( + not part or part in {".", ".."} or re.fullmatch(r"[A-Za-z0-9._-]+", part) is None + for part in parts): + raise TraceError(f"trace path is not canonical: {relative}") + if PurePosixPath(*parts).as_posix() != relative: + raise TraceError(f"trace path is not canonical: {relative}") + return tuple(parts) + + +def _safe_bundle_file(root: Path, relative: str) -> Path: + parts = _bundle_path_parts(relative) + candidate = root + for part in parts: + candidate = candidate / part + if candidate.is_symlink(): + raise TraceError(f"trace path must not use symlinks: {relative}") + try: + candidate.resolve().relative_to(root) + except ValueError as error: + raise TraceError(f"trace path is outside the bundle: {relative}") from error + if not candidate.is_file(): + raise TraceError(f"trace bundle file is missing or not regular: {relative}") + return candidate + + +def _read_bundle_file( + root: Path, + relative: str, + *, + retain: bool) -> tuple[BundleFileReceipt, bytes | None]: + path = _safe_bundle_file(root, relative) + flags = os.O_RDONLY + if hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as error: + raise TraceError(f"cannot open trace bundle file {relative}: {error}") from error + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise TraceError(f"trace bundle file is not regular: {relative}") + if getattr(before, "st_nlink", 1) != 1: + raise TraceError(f"trace bundle file must not be hard linked: {relative}") + digest = hashlib.sha256() + chunks = [] if retain else None + byte_count = 0 + while True: + chunk = os.read(descriptor, 8 * 1024 * 1024) + if not chunk: + break + digest.update(chunk) + byte_count += len(chunk) + if chunks is not None: + chunks.append(chunk) + after = os.fstat(descriptor) + path_stat = os.stat(path, follow_symlinks=False) + except OSError as error: + raise TraceError(f"cannot read trace bundle file {relative}: {error}") from error + finally: + os.close(descriptor) + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + identity_path = ( + path_stat.st_dev, + path_stat.st_ino, + path_stat.st_size, + path_stat.st_mtime_ns, + path_stat.st_ctime_ns, + ) + if identity_before != identity_after or identity_after != identity_path or byte_count != after.st_size: + raise TraceError(f"trace bundle file changed while reading: {relative}") + receipt = BundleFileReceipt( + device=after.st_dev, + inode=after.st_ino, + byte_count=byte_count, + modified_ns=after.st_mtime_ns, + changed_ns=after.st_ctime_ns, + sha256=digest.hexdigest(), + ) + return receipt, b"".join(chunks) if chunks is not None else None + + +def _read_canonical_json( + root: Path, + relative: str) -> tuple[dict[str, Any], bytes, BundleFileReceipt]: + receipt, retained = _read_bundle_file(root, relative, retain=True) + assert retained is not None + try: + data = retained + text = data.decode("ascii") + record = strict_json_loads(text) + except (UnicodeError, TraceError) as error: + raise TraceError(f"cannot read canonical JSON {relative}: {error}") from error + if not isinstance(record, dict) or data != _canonical_json_bytes(record): + raise TraceError(f"trace JSON is not canonical: {relative}") + return record, data, receipt + + +def _read_canonical_jsonl( + root: Path, + relative: str) -> tuple[list[dict[str, Any]], bytes, BundleFileReceipt]: + receipt, retained = _read_bundle_file(root, relative, retain=True) + assert retained is not None + data = retained + if not data or not data.endswith(b"\n"): + raise TraceError(f"trace JSONL is empty or truncated: {relative}") + records = [] + for line_number, raw in enumerate(data.splitlines(keepends=True), 1): + try: + text = raw.decode("ascii") + record = strict_json_loads(text) + except (UnicodeError, TraceError) as error: + raise TraceError(f"invalid JSONL at {relative}:{line_number}: {error}") from error + if not isinstance(record, dict) or raw != _canonical_json_bytes(record): + raise TraceError(f"trace JSONL is not canonical: {relative}:{line_number}") + records.append(record) + return records, data, receipt + + +def _bundle_domain( + root: Path, +) -> tuple[ + bytes, + dict[str, Any], + list[dict[str, Any]], + dict[str, BundleFileReceipt], + dict[str, bytes], +]: + if root.is_symlink(): + raise TraceError("trace root must not be a symlink") + try: + canonical_root = root.resolve(strict=True) + except OSError as error: + raise TraceError(f"cannot resolve trace root: {error}") from error + if not canonical_root.is_dir(): + raise TraceError("trace root is not a directory") + + manifest, manifest_bytes, manifest_receipt = _read_canonical_json(canonical_root, MANIFEST_NAME) + events, events_bytes, events_receipt = _read_canonical_jsonl(canonical_root, EVENTS_NAME) + expected_paths = {MANIFEST_NAME, EVENTS_NAME} + receipts = { + MANIFEST_NAME: manifest_receipt, + EVENTS_NAME: events_receipt, + } + contents = { + MANIFEST_NAME: manifest_bytes, + EVENTS_NAME: events_bytes, + } + + for event in events: + blob = event.get("blob") + if not isinstance(blob, str): + raise TraceError("event blob reference is invalid") + expected_paths.add(blob) + + prompt = manifest.get("prompt") + provenance = prompt.get("provenance") if isinstance(prompt, dict) else None + if not isinstance(provenance, dict) or not isinstance(provenance.get("path"), str): + raise TraceError("prompt provenance reference is missing") + expected_paths.add(provenance["path"]) + + audits = manifest.get("audits") + if not isinstance(audits, dict): + raise TraceError("manifest audit envelope is invalid") + audit_paths = [] + referenced_metadata_paths = {provenance["path"]} + for phase in ("pre", "post"): + phase_audits = audits.get(phase) + if not isinstance(phase_audits, dict): + raise TraceError(f"manifest {phase} audit set is invalid") + for reference in phase_audits.values(): + if not isinstance(reference, dict) or not isinstance(reference.get("path"), str): + raise TraceError(f"manifest {phase} audit reference is invalid") + if reference["path"] in referenced_metadata_paths: + raise TraceError(f"trace metadata path is referenced more than once: {reference['path']}") + referenced_metadata_paths.add(reference["path"]) + audit_paths.append(reference["path"]) + expected_paths.add(reference["path"]) + + for relative in audit_paths: + record, data, receipt = _read_canonical_json(canonical_root, relative) + receipts[relative] = receipt + contents[relative] = data + audit = record.get("data", {}).get("audit") + if isinstance(audit, dict) and isinstance(audit.get("path"), str): + audit_jsonl = audit["path"] + if audit_jsonl in referenced_metadata_paths: + raise TraceError(f"trace metadata path is referenced more than once: {audit_jsonl}") + referenced_metadata_paths.add(audit_jsonl) + expected_paths.add(audit_jsonl) + _records, jsonl_data, jsonl_receipt = _read_canonical_jsonl(canonical_root, audit_jsonl) + receipts[audit_jsonl] = jsonl_receipt + contents[audit_jsonl] = jsonl_data + + _provenance, provenance_data, provenance_receipt = _read_canonical_json( + canonical_root, + provenance["path"], + ) + receipts[provenance["path"]] = provenance_receipt + contents[provenance["path"]] = provenance_data + + actual_paths = set() + try: + for path in canonical_root.rglob("*"): + relative = path.relative_to(canonical_root).as_posix() + if path.is_symlink(): + raise TraceError(f"trace bundle contains a symlink: {relative}") + if path.is_file(): + if relative != SIGNATURE_NAME: + actual_paths.add(relative) + elif not path.is_dir(): + raise TraceError(f"trace bundle contains a nonregular entry: {relative}") + except OSError as error: + raise TraceError(f"cannot enumerate trace bundle: {error}") from error + if actual_paths != expected_paths: + missing = sorted(expected_paths - actual_paths) + extra = sorted(actual_paths - expected_paths) + detail = [] + if missing: + detail.append("missing " + ", ".join(missing)) + if extra: + detail.append("unexpected " + ", ".join(extra)) + raise TraceError(f"trace signed file set is invalid: {'; '.join(detail)}") + + records = [] + for relative in sorted(expected_paths, key=lambda value: value.encode("ascii")): + _bundle_path_parts(relative) + data = contents.get(relative) + if data is not None: + byte_count = len(data) + digest = sha256_bytes(data) + else: + receipt, _retained = _read_bundle_file(canonical_root, relative, retain=False) + receipts[relative] = receipt + byte_count = receipt.byte_count + digest = receipt.sha256 + records.append({ + "path": relative, + "byte_count": byte_count, + "sha256": digest, + }) + domain = SEAL_DOMAIN_PREFIX + _canonical_json_bytes(records) + return domain, manifest, events, receipts, contents + + +def _validate_unsealed_bundle( + root: Path, + manifest: dict[str, Any], + events: list[dict[str, Any]], + receipts: dict[str, BundleFileReceipt], + contents: dict[str, bytes], + verifier: TraceVerifier) -> None: + bundle = object.__new__(_TRACE_BUNDLE_TYPE) + bundle.root = root + bundle.signer_principal = verifier.principal + bundle.verifier = verifier + bundle.manifest = manifest + bundle._file_receipts = receipts + bundle._retained_files = contents + bundle._validate_manifest() + bundle.events = bundle._validate_sealed_events(events, True) + if bundle.manifest.get("event_count") != len(bundle.events): + raise TraceError("manifest event_count mismatch") + bundle._validate_coverage() + + +def seal_bundle( + root: Path, + *, + private_key: Path, + principal: str, + expected_lane: str, + expected_challenge: str, + expected_run_id: str, + candidate_exporter_policies: dict[str, dict[str, Any]] = APPROVED_CANDIDATE_EXPORTERS, + ds4_exporter_policies: dict[str, dict[str, Any]] = APPROVED_DS4_EXPORTERS, + prompt_builder_policies: dict[str, dict[str, Any]] = APPROVED_PROMPT_BUILDERS, + expected_candidate_exporter_policy_id: str | None = None, + expected_ds4_exporter_policy_id: str | None = None, + expected_prompt_builder_policy_id: str = "", + expected_approval_policy_sha256: str = "", + expected_verifier_revision: str = "", + verification_unix: int | None = None, + trusted_signers: dict[str, dict[str, str]] = APPROVED_TRACE_SIGNERS, + ssh_keygen: Path | None = None) -> str: + root = root.resolve() + executable, _public_key = validate_signing_identity( + private_key, + principal, + trusted_signers=trusted_signers, + ssh_keygen=ssh_keygen, + forbidden_root=root, + ) + key_path = private_key.resolve() + signature_path = root / SIGNATURE_NAME + if signature_path.exists() or signature_path.is_symlink(): + raise TraceError("trace signature envelope already exists") + domain, manifest, events, receipts, contents = _bundle_domain(root) + verification_time = int(time.time()) if verification_unix is None else verification_unix + policy = _signer_policy(trusted_signers, principal) + validate_execution_authorization( + manifest, + policy=policy, + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=verification_time, + candidate_exporter_policies=candidate_exporter_policies, + ds4_exporter_policies=ds4_exporter_policies, + prompt_builder_policies=prompt_builder_policies, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=expected_approval_policy_sha256, + expected_verifier_revision=expected_verifier_revision, + ) + _validate_unsealed_bundle( + root, + manifest, + events, + receipts, + contents, + TraceVerifier( + principal=principal, + trusted_signers=trusted_signers, + ssh_keygen=executable, + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + verification_unix=verification_time, + candidate_exporter_policies=candidate_exporter_policies, + ds4_exporter_policies=ds4_exporter_policies, + prompt_builder_policies=prompt_builder_policies, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + expected_approval_policy_sha256=expected_approval_policy_sha256, + expected_verifier_revision=expected_verifier_revision, + ), + ) + try: + with tempfile.TemporaryDirectory(prefix="dsv41-trace-sign-") as signing_temp: + signing_root = Path(signing_temp).resolve() + if os.name != "nt": + signing_root.chmod(0o700) + domain_path = signing_root / "bundle-domain" + signature_file = signing_root / "bundle-domain.sig" + with domain_path.open("xb") as stream: + stream.write(domain) + stream.flush() + os.fsync(stream.fileno()) + if os.name != "nt": + domain_path.chmod(0o600) + if signature_file.exists() or signature_file.is_symlink(): + raise TraceError("temporary trace signature output already exists") + result = subprocess.run( + [ + str(executable), + "-Y", "sign", + "-f", str(key_path), + "-n", SEAL_NAMESPACE, + str(domain_path), + ], + stdin=subprocess.DEVNULL, + check=False, + capture_output=True, + timeout=30, + env=_ssh_environment(), + ) + if result.returncode != 0: + raise TraceError("trusted ssh-keygen failed to sign trace bundle") + signature_receipt, signature_bytes = _read_bundle_file( + signing_root, + signature_file.name, + retain=True, + ) + if signature_receipt.byte_count == 0 or signature_receipt.sha256 != sha256_bytes(signature_bytes or b""): + raise TraceError("trusted ssh-keygen did not create a stable signature") + assert signature_bytes is not None + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot sign trace bundle: {error}") from error + try: + signature = signature_bytes.decode("ascii") + except UnicodeError as error: + raise TraceError("trace signature is not ASCII") from error + if not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or not signature.endswith( + "-----END SSH SIGNATURE-----\n"): + raise TraceError("trusted ssh-keygen returned an invalid signature") + envelope = { + "format": SEAL_FORMAT, + "version": SEAL_VERSION, + "namespace": SEAL_NAMESPACE, + "principal": principal, + "domain_sha256": sha256_bytes(domain), + "signature": signature, + } + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=root, + prefix=".bundle-signature.", + delete=False) as stream: + temporary_path = Path(stream.name) + stream.write(_canonical_json_bytes(envelope)) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary_path, signature_path) + except FileExistsError as error: + raise TraceError("trace signature envelope already exists") from error + except OSError as error: + raise TraceError(f"cannot install trace signature envelope: {error}") from error + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + return envelope["domain_sha256"] + + +def verify_bundle_seal( + root: Path, + verifier: TraceVerifier, +) -> tuple[ + dict[str, Any], + list[dict[str, Any]], + str, + dict[str, BundleFileReceipt], + dict[str, bytes], +]: + policy = _signer_policy(verifier.trusted_signers, verifier.principal) + approved_key = _normalize_public_key(policy["public_key"]) + executable = _validate_ssh_keygen(verifier.ssh_keygen) + root = root.resolve() + envelope, _envelope_bytes, _envelope_receipt = _read_canonical_json(root, SIGNATURE_NAME) + _require_exact_keys( + envelope, + {"format", "version", "namespace", "principal", "domain_sha256", "signature"}, + "trace signature envelope", + ) + if envelope.get("format") != SEAL_FORMAT or envelope.get("version") != SEAL_VERSION or ( + envelope.get("namespace") != SEAL_NAMESPACE): + raise TraceError("trace signature envelope version is invalid") + if envelope.get("principal") != verifier.principal: + raise TraceError("trace signature principal differs from the externally expected signer") + signature = envelope.get("signature") + if not isinstance(signature, str) or not signature.startswith("-----BEGIN SSH SIGNATURE-----\n") or ( + not signature.endswith("-----END SSH SIGNATURE-----\n")): + raise TraceError("trace signature envelope contains an invalid signature") + domain, manifest, events, receipts, contents = _bundle_domain(root) + validate_execution_authorization( + manifest, + policy=policy, + expected_lane=verifier.expected_lane, + expected_challenge=verifier.expected_challenge, + expected_run_id=verifier.expected_run_id, + verification_unix=verifier.verification_unix, + candidate_exporter_policies=verifier.candidate_exporter_policies, + ds4_exporter_policies=verifier.ds4_exporter_policies, + prompt_builder_policies=verifier.prompt_builder_policies, + expected_candidate_exporter_policy_id=verifier.expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=verifier.expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=verifier.expected_prompt_builder_policy_id, + expected_approval_policy_sha256=verifier.expected_approval_policy_sha256, + expected_verifier_revision=verifier.expected_verifier_revision, + seen_run_ids=verifier.seen_run_ids, + ) + domain_sha256 = sha256_bytes(domain) + if envelope.get("domain_sha256") != domain_sha256: + raise TraceError("trace signed-domain SHA-256 mismatch") + with tempfile.TemporaryDirectory(prefix="dsv41-trace-verify-") as temp: + temporary = Path(temp) + allowed_signers = temporary / "allowed_signers" + signature_file = temporary / "signature" + allowed_signers.write_text( + f"{verifier.principal} {approved_key}\n", + encoding="ascii", + ) + signature_file.write_text(signature, encoding="ascii") + try: + result = subprocess.run( + [ + str(executable), + "-Y", "verify", + "-f", str(allowed_signers), + "-I", verifier.principal, + "-n", SEAL_NAMESPACE, + "-s", str(signature_file), + ], + input=domain, + check=False, + capture_output=True, + timeout=30, + env=_ssh_environment(), + ) + except (OSError, subprocess.SubprocessError) as error: + raise TraceError(f"cannot verify trace bundle signature: {error}") from error + if result.returncode != 0: + raise TraceError("trace bundle signature verification failed") + return manifest, events, domain_sha256, receipts, contents + + +def _require_exact_keys(record: dict[str, Any], keys: set[str], label: str) -> None: + missing = sorted(keys - set(record)) + extra = sorted(set(record) - keys) + if missing or extra: + detail = [] + if missing: + detail.append("missing " + ", ".join(missing)) + if extra: + detail.append("unexpected " + ", ".join(extra)) + raise TraceError(f"{label} fields are invalid: {'; '.join(detail)}") + + +def validate_accelerator_attestation(runtime: str, accelerator: Any) -> dict[str, Any]: + if not isinstance(accelerator, dict): + raise TraceError("manifest accelerator attestation is invalid") + common = { + "format", + "version", + "runtime_kind", + "platform", + "backend", + "backend_device", + "backend_description", + "architecture", + "source", + } + if runtime == "llama.cpp": + _require_exact_keys( + accelerator, + common | {"pci_device_id", "kfd_node", "gpu_id", "gfx_target_version"}, + "llama.cpp accelerator attestation", + ) + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "backend": "ROCm", + "backend_device": "ROCm0", + "architecture": "gfx1151", + "gfx_target_version": 110501, + "source": "linux-kfd-sysfs", + } + for key, value in expected.items(): + if accelerator.get(key) != value: + raise TraceError(f"llama.cpp accelerator {key} mismatch") + if re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", + accelerator.get("pci_device_id", "")) is None: + raise TraceError("llama.cpp accelerator PCI identity is invalid") + if not isinstance(accelerator.get("kfd_node"), str) or not accelerator["kfd_node"].isdigit(): + raise TraceError("llama.cpp accelerator KFD node is invalid") + if type(accelerator.get("gpu_id")) is not int or accelerator["gpu_id"] <= 0: + raise TraceError("llama.cpp accelerator GPU identity is invalid") + elif runtime == "ds4": + _require_exact_keys( + accelerator, + common | { + "metal_registry_id", + "recommended_max_working_set_bytes", + "unified_memory", + }, + "ds4 accelerator attestation", + ) + expected = { + "format": "dsv41-accelerator-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "backend": "Metal", + "source": "metal-device-query", + "unified_memory": True, + } + for key, value in expected.items(): + if accelerator.get(key) != value: + raise TraceError(f"ds4 accelerator {key} mismatch") + if type(accelerator.get("unified_memory")) is not bool: + raise TraceError("ds4 accelerator unified-memory identity is invalid") + registry_id = accelerator.get("metal_registry_id") + if type(registry_id) is not int or registry_id <= 0: + raise TraceError("ds4 accelerator Metal registry identity is invalid") + working_set = accelerator.get("recommended_max_working_set_bytes") + if type(working_set) is not int or working_set <= 0: + raise TraceError("ds4 accelerator working-set identity is invalid") + else: + raise TraceError(f"unsupported runtime accelerator attestation: {runtime}") + for key in ("backend_device", "backend_description", "architecture"): + if not isinstance(accelerator.get(key), str) or not accelerator[key]: + raise TraceError(f"manifest accelerator {key} is invalid") + return accelerator + + +def validate_storage_attestation(runtime: str, item: Any) -> dict[str, Any]: + if not isinstance(item, dict): + raise TraceError("storage attestation is invalid") + common = { + "format", + "version", + "runtime_kind", + "platform", + "storage_kind", + "resolved_path", + "existing_path", + "mount_point", + "filesystem_type", + "source", + } + if runtime == "llama.cpp": + _require_exact_keys( + item, + common | { + "mount_source", + "device_number", + "block_device_path", + "nvme_device", + "rotational", + }, + "llama.cpp storage attestation", + ) + expected = { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "strix-rocm", + "platform": "linux", + "storage_kind": "linux-nvme", + "source": "linux-mountinfo-sysfs", + "rotational": False, + } + for key, value in expected.items(): + if item.get(key) != value: + raise TraceError(f"llama.cpp storage {key} mismatch") + if type(item.get("rotational")) is not bool: + raise TraceError("llama.cpp storage rotational identity is invalid") + if not isinstance(item.get("nvme_device"), str) or re.fullmatch( + r"nvme[0-9]+(?:c[0-9]+)?n[0-9]+", item["nvme_device"]) is None: + raise TraceError("llama.cpp storage NVMe device identity is invalid") + if not isinstance(item.get("mount_source"), str) or not item["mount_source"].startswith("/dev/"): + raise TraceError("llama.cpp storage mount source is not a local block device") + if re.fullmatch(r"[0-9]+:[0-9]+", item.get("device_number", "")) is None: + raise TraceError("llama.cpp storage device number is invalid") + block_device_path = item.get("block_device_path") + if not isinstance(block_device_path, str) or item["nvme_device"] not in Path(block_device_path).parts: + raise TraceError("llama.cpp storage block device ancestry is invalid") + elif runtime == "ds4": + _require_exact_keys( + item, + common | { + "device_identifier", + "parent_whole_disk", + "bus_protocol", + "filesystem_device", + "internal", + "solid_state", + }, + "ds4 storage attestation", + ) + expected = { + "format": "dsv41-storage-attestation", + "version": 2, + "runtime_kind": "apple-metal", + "platform": "macos", + "storage_kind": "darwin-local-solid-state", + "source": "diskutil-info-plist", + "internal": True, + "solid_state": True, + } + for key, value in expected.items(): + if item.get(key) != value: + raise TraceError(f"ds4 storage {key} mismatch") + if type(item.get("internal")) is not bool or type(item.get("solid_state")) is not bool: + raise TraceError("ds4 storage media identity is invalid") + for key in ("device_identifier", "parent_whole_disk", "bus_protocol"): + if not isinstance(item.get(key), str) or not item[key]: + raise TraceError(f"ds4 storage {key} is invalid") + if item["bus_protocol"].lower() not in {"nvme", "apple fabric"}: + raise TraceError("ds4 storage is not NVMe-backed") + if type(item.get("filesystem_device")) is not int or item["filesystem_device"] < 0: + raise TraceError("ds4 storage filesystem device identity is invalid") + else: + raise TraceError(f"unsupported runtime storage attestation: {runtime}") + for path_key in ("resolved_path", "existing_path", "mount_point"): + value = item.get(path_key) + if not isinstance(value, str) or not value.startswith("/"): + raise TraceError(f"storage {path_key} is invalid") + if not isinstance(item.get("filesystem_type"), str) or not item["filesystem_type"]: + raise TraceError("storage filesystem type is invalid") + try: + Path(item["resolved_path"]).relative_to(Path(item["mount_point"])) + Path(item["existing_path"]).relative_to(Path(item["mount_point"])) + except ValueError as error: + raise TraceError("storage mount ancestry is invalid") from error + return item + + +def validate_host_attestation(host: Any) -> dict[str, Any]: + if not isinstance(host, dict): + raise TraceError("ds4 host attestation is invalid") + _require_exact_keys( + host, + { + "format", + "version", + "runtime_kind", + "platform", + "machine", + "hardware_model", + "os_version", + "memory_bytes", + "source", + }, + "ds4 host attestation", + ) + expected = { + "format": "dsv41-host-attestation", + "version": 1, + "runtime_kind": "apple-metal", + "platform": "macos", + "machine": "arm64", + "source": "darwin-sysctl", + } + for key, value in expected.items(): + if host.get(key) != value: + raise TraceError(f"ds4 host {key} mismatch") + for key in ("hardware_model", "os_version"): + if not isinstance(host.get(key), str) or not host[key]: + raise TraceError(f"ds4 host {key} is invalid") + if type(host.get("memory_bytes")) is not int or host["memory_bytes"] < 128 * 1024 * 1024 * 1024: + raise TraceError("ds4 host memory is below 128 GiB") + return host + + +WATCHDOG_STATE_KEYS = { + "total_bytes", + "available_bytes", + "used_bytes", + "swap_entries", + "peak_used_bytes", + "child_pid", + "child_status", + "child_returncode", + "process_group_id", + "process_group_status", + "threshold_reason", +} +WATCHDOG_REQUIRED_ERROR_CLASSIFICATIONS = { + "configuration_error", + "internal_error", + "launch_error", + "lease_error", + "termination_timeout", +} +WATCHDOG_OPTIONAL_ERROR_CLASSIFICATIONS = { + "procfs_error", + "signal_error", +} + + +def validate_watchdog_event(event: Any) -> dict[str, Any]: + if not isinstance(event, dict): + raise TraceError("watchdog JSONL event is not an object") + event_name = event.get("event") + required = {"timestamp", "event"} | WATCHDOG_STATE_KEYS + optional: set[str] = set() + if event_name == "preflight": + required |= {"soft_bytes", "emergency_bytes", "strict_ceiling_bytes"} + elif event_name == "child_started": + required.add("command") + elif event_name == "sample": + pass + elif event_name == "process_group_signal": + required.add("signal") + optional.add("grace_deadline_monotonic") + elif event_name == "final": + required |= {"classification", "exit_code"} + optional |= {"error", "secondary_errors"} + else: + raise TraceError(f"watchdog JSONL event name is invalid: {event_name}") + missing = sorted(required - set(event)) + unexpected = sorted(set(event) - required - optional) + if missing or unexpected: + details = [] + if missing: + details.append("missing " + ", ".join(missing)) + if unexpected: + details.append("unexpected " + ", ".join(unexpected)) + raise TraceError("watchdog JSONL event fields are invalid: " + "; ".join(details)) + if not isinstance(event["timestamp"], str) or re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z", + event["timestamp"]) is None: + raise TraceError("watchdog JSONL timestamp is invalid") + for key in ("total_bytes", "available_bytes", "used_bytes", "swap_entries", "peak_used_bytes"): + if event[key] is not None and (type(event[key]) is not int or event[key] < 0): + raise TraceError(f"watchdog JSONL {key} is invalid") + for key in ("child_pid", "process_group_id"): + if event[key] is not None and (type(event[key]) is not int or event[key] <= 0): + raise TraceError(f"watchdog JSONL {key} is invalid") + if event["child_returncode"] is not None and type(event["child_returncode"]) is not int: + raise TraceError("watchdog JSONL child_returncode is invalid") + if event["child_status"] not in {"not_started", "running", "signaled", "exited"}: + raise TraceError("watchdog JSONL child_status is invalid") + if event["process_group_status"] not in { + "not_created", "active", "leader_exited", "signal_error", + "termination_timeout", "missing", "sighup_sent", "sigint_sent", + "sigterm_sent", "sigkill_sent", "sigkill_timeout"}: + raise TraceError("watchdog JSONL process_group_status is invalid") + if not isinstance(event["threshold_reason"], str) or not event["threshold_reason"]: + raise TraceError("watchdog JSONL threshold_reason is invalid") + if event_name == "preflight": + for key in ("soft_bytes", "emergency_bytes", "strict_ceiling_bytes"): + if type(event[key]) is not int or event[key] <= 0: + raise TraceError(f"watchdog JSONL {key} is invalid") + elif event_name == "child_started": + if not isinstance(event["command"], list) or not event["command"] or ( + not all(isinstance(value, str) and value for value in event["command"])): + raise TraceError("watchdog JSONL command is invalid") + elif event_name == "process_group_signal": + if event["signal"] not in {"SIGHUP", "SIGINT", "SIGTERM", "SIGKILL"}: + raise TraceError("watchdog JSONL signal is invalid") + deadline = event.get("grace_deadline_monotonic") + if deadline is not None and ( + event["signal"] != "SIGTERM" or + not isinstance(deadline, (int, float)) or isinstance(deadline, bool) or + not math.isfinite(deadline)): + raise TraceError("watchdog JSONL grace deadline is invalid") + elif event_name == "final": + if event["classification"] not in { + "child_exit", "soft_limit", "grace_timeout", "swap_appeared", + "emergency_limit", "procfs_error", "signal_error", "termination_timeout", + "lease_error", "parent_signal", "internal_error", "launch_error", + "configuration_error", "startup_swap_active", "startup_emergency_limit", + "startup_soft_limit"}: + raise TraceError("watchdog JSONL classification is invalid") + if type(event["exit_code"]) is not int: + raise TraceError("watchdog JSONL exit code is invalid") + classification = event["classification"] + requires_error = classification in WATCHDOG_REQUIRED_ERROR_CLASSIFICATIONS + allows_error = requires_error or classification in WATCHDOG_OPTIONAL_ERROR_CLASSIFICATIONS + if requires_error and "error" not in event: + raise TraceError("watchdog JSONL error presence does not match classification") + if not allows_error and "error" in event: + raise TraceError("watchdog JSONL error presence does not match classification") + if "error" in event and (not isinstance(event["error"], str) or not event["error"]): + raise TraceError("watchdog JSONL error is invalid") + if "secondary_errors" in event: + secondary_errors = event["secondary_errors"] + if classification != "signal_error" or "error" not in event: + raise TraceError("watchdog JSONL secondary errors require a primary signal error") + if not isinstance(secondary_errors, list) or not secondary_errors: + raise TraceError("watchdog JSONL secondary errors are invalid") + for secondary_error in secondary_errors: + _require_exact_keys( + secondary_error, {"component", "detail"}, "watchdog JSONL secondary error") + if secondary_error["component"] not in {"audit", "lease", "stderr"} or ( + not isinstance(secondary_error["detail"], str) or not secondary_error["detail"]): + raise TraceError("watchdog JSONL secondary error is invalid") + return event + + +def element_count(shape: Iterable[int]) -> int: + count = 1 + for dim in shape: + if type(dim) is not int or dim <= 0: + raise TraceError(f"shape dimension must be a nonzero positive integer: {dim!r}") + count *= dim + return count + + +def expected_bytes(event: dict[str, Any]) -> int: + dtype = event.get("dtype") + if dtype not in DTYPE_SIZES: + raise TraceError(f"unsupported dtype: {dtype!r}") + shape = event.get("shape") + if not isinstance(shape, list): + raise TraceError("event shape must be an array") + return element_count(shape) * DTYPE_SIZES[dtype] + + +def event_key(event: dict[str, Any]) -> tuple[Any, ...]: + return ( + event.get("phase", ""), + int(event.get("step", -1)), + int(event.get("token_start", -1)), + event.get("layer"), + event.get("component", ""), + ) + + +def event_order(key: tuple[Any, ...]) -> tuple[Any, ...]: + phase_order = {"input": 0, "prefill": 1, "decode": 2} + component_order = { + "prompt.bytes": 0, + "prompt.tokens": 1, + "engram.row_ids": 2, + "expert.ids": 3, + "expert.weights": 4, + "attn.source": 5, + "attn.candidate_blocks": 6, + "attn.candidates": 7, + "decode.greedy_token": 8, + "logits.prefill": 9, + "logits.decode": 10, + } + return ( + phase_order.get(key[0], 99), + key[2], + key[1], + -1 if key[3] is None else key[3], + component_order.get(key[4], 99), + key[4], + ) + + +def classify(component: str) -> str: + if component == "prompt.tokens": + return "tokenizer" + if component == "engram.row_ids": + return "engram_row" + if component == "expert.ids": + return "routing_original_expert" + if component == "expert.weights": + return "routing_weight" + if component.startswith("attn.candidate"): + return "attention_candidate" + if component == "attn.source": + return "attention_source" + if component == "logits.prefill": + return "prefill_logits" + if component == "logits.decode": + return "decode_logits" + if component == "decode.greedy_token": + return "decode_token" + return "trace_data" + + +def validate_event(event: dict[str, Any]) -> None: + required = { + "trace_version", + "component", + "phase", + "step", + "token_start", + "token_count", + "layer", + "dtype", + "shape", + "byte_order", + "byte_count", + "sha256", + "blob", + } + missing = sorted(required - event.keys()) + if missing: + raise TraceError(f"event is missing fields: {', '.join(missing)}") + allowed = set(required) + if event.get("component") == "expert.ids": + allowed.add("semantic_id_space") + extra = sorted(set(event) - allowed) + if extra: + raise TraceError(f"event has unexpected fields: {', '.join(extra)}") + if event["trace_version"] != TRACE_VERSION: + raise TraceError(f"unsupported event version: {event['trace_version']!r}") + if event["byte_order"] != "little": + raise TraceError("trace blobs must use little-endian byte order") + if event["phase"] not in ("input", "prefill", "decode"): + raise TraceError("event phase is invalid") + if type(event["step"]) is not int or event["step"] < 0: + raise TraceError("event step is invalid") + if type(event["token_start"]) is not int or event["token_start"] < 0: + raise TraceError("event token_start is invalid") + if type(event["token_count"]) is not int or event["token_count"] <= 0: + raise TraceError("event token_count is invalid") + if event["layer"] is not None and (type(event["layer"]) is not int or event["layer"] < 0): + raise TraceError("event layer is invalid") + if not isinstance(event["shape"], list) or not event["shape"] or any(dim <= 0 for dim in event["shape"]): + raise TraceError("event shape dimensions must be nonzero") + if event["byte_count"] != expected_bytes(event): + raise TraceError("event byte_count does not match dtype and shape") + digest = event["sha256"] + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError("event sha256 is invalid") + if event["blob"] != f"{BLOBS_DIR}/{digest}.bin": + raise TraceError("event blob path is not content addressed") + component = event["component"] + if component == "expert.ids": + if event.get("semantic_id_space") != "original": + raise TraceError("expert.ids must contain original expert IDs, not cache slot IDs") + if "slot" in component or event.get("semantic_id_space") == "cache_slot": + raise TraceError("cache slot IDs are forbidden in correctness traces") + + +class TraceBundleWriter: + def __init__(self, root: Path, manifest: dict[str, Any]): + self.root = root + self.blobs = root / BLOBS_DIR + if root.exists() and any(root.iterdir()): + raise TraceError(f"trace output directory is not empty: {root}") + self.blobs.mkdir(parents=True, exist_ok=True) + self.events = (root / EVENTS_NAME).open("x", encoding="ascii", newline="\n") + self.manifest = dict(manifest) + self.manifest["trace_format"] = TRACE_FORMAT + self.manifest["trace_version"] = TRACE_VERSION + self.event_count = 0 + + def add_event( + self, + *, + component: str, + phase: str, + step: int, + token_start: int, + token_count: int, + layer: int | None, + dtype: str, + shape: list[int], + data: bytes, + semantic_id_space: str | None = None, + ) -> dict[str, Any]: + digest = sha256_bytes(data) + blob_rel = f"{BLOBS_DIR}/{digest}.bin" + event = { + "trace_version": TRACE_VERSION, + "component": component, + "phase": phase, + "step": step, + "token_start": token_start, + "token_count": token_count, + "layer": layer, + "dtype": dtype, + "shape": shape, + "byte_order": "little", + "byte_count": len(data), + "sha256": digest, + "blob": blob_rel, + } + if semantic_id_space is not None: + event["semantic_id_space"] = semantic_id_space + validate_event(event) + blob_path = self.root / blob_rel + if not blob_path.exists(): + temp = blob_path.with_suffix(".tmp") + temp.write_bytes(data) + os.replace(temp, blob_path) + self.events.write(canonical_json(event) + "\n") + self.events.flush() + self.event_count += 1 + return event + + def close(self) -> None: + if self.events.closed: + return + self.events.close() + self.manifest["event_count"] = self.event_count + manifest_path = self.root / MANIFEST_NAME + temp = manifest_path.with_suffix(".tmp") + temp.write_text(canonical_json(self.manifest) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + def __enter__(self) -> "TraceBundleWriter": + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + self.events.close() + if exc_type is None: + self.manifest["event_count"] = self.event_count + manifest_path = self.root / MANIFEST_NAME + temp = manifest_path.with_suffix(".tmp") + temp.write_text(canonical_json(self.manifest) + "\n", encoding="ascii") + os.replace(temp, manifest_path) + + +class TraceBundle: + def __init__( + self, + root: Path, + verify_blobs: bool = True, + *, + verifier: TraceVerifier | None = None, + signer_principal: str | None = None, + expected_lane: str | None = None, + expected_challenge: str | None = None, + expected_run_id: str | None = None, + expected_candidate_exporter_policy_id: str | None = None, + expected_ds4_exporter_policy_id: str | None = None, + expected_prompt_builder_policy_id: str | None = None, + verification_unix: int | None = None, + seen_run_ids: set[str] | None = None): + if root.is_symlink(): + raise TraceError("trace root must not be a symlink") + self.root = root.resolve() + if verifier is None: + if None in ( + signer_principal, expected_lane, expected_challenge, expected_run_id, + expected_prompt_builder_policy_id): + raise TraceError( + "external signer, lane, challenge, run ID, and prompt builder approval are required") + if expected_lane == CANDIDATE_LANE and expected_candidate_exporter_policy_id is None: + raise TraceError("external candidate exporter approval is required") + if expected_lane == ORACLE_LANE and expected_ds4_exporter_policy_id is None: + raise TraceError("external ds4 exporter approval is required") + verifier = TraceVerifier.production( + signer_principal, + expected_lane=expected_lane, + expected_challenge=expected_challenge, + expected_run_id=expected_run_id, + expected_candidate_exporter_policy_id=expected_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=expected_prompt_builder_policy_id, + verification_unix=verification_unix, + seen_run_ids=seen_run_ids, + ) + elif any(value is not None for value in ( + signer_principal, expected_lane, expected_challenge, expected_run_id, + expected_candidate_exporter_policy_id, expected_ds4_exporter_policy_id, + expected_prompt_builder_policy_id, + verification_unix, seen_run_ids)): + raise TraceError("trace verifier cannot be combined with separate verification inputs") + self.signer_principal = verifier.principal + self.verifier = verifier + ( + self.manifest, + sealed_events, + self.seal_sha256, + self._file_receipts, + self._retained_files, + ) = verify_bundle_seal(self.root, verifier) + if self.manifest.get("trace_format") != TRACE_FORMAT: + raise TraceError("manifest trace_format mismatch") + if self.manifest.get("trace_version") != TRACE_VERSION: + raise TraceError("manifest trace_version mismatch") + self._validate_manifest() + self.events = self._validate_sealed_events(sealed_events, verify_blobs) + if self.manifest.get("event_count") != len(self.events): + raise TraceError("manifest event_count mismatch") + self._validate_coverage() + + def _validate_sealed_events( + self, + sealed_events: list[dict[str, Any]], + verify_blobs: bool) -> list[dict[str, Any]]: + result = [] + for line_number, event in enumerate(sealed_events, 1): + validate_event(event) + if verify_blobs: + data = self.read_blob(event) + if len(data) != event["byte_count"]: + raise TraceError(f"truncated blob for event line {line_number}") + if sha256_bytes(data) != event["sha256"]: + raise TraceError(f"corrupt blob for event line {line_number}") + result.append(event) + return result + + def _validate_manifest(self) -> None: + if not isinstance(self.manifest.get("runtime"), str) or not self.manifest["runtime"]: + raise TraceError("manifest runtime is invalid") + if self.manifest["runtime"] not in ("ds4", "llama.cpp"): + raise TraceError("manifest runtime must be ds4 or llama.cpp") + top_level = { + "trace_format", + "trace_version", + "event_count", + "runtime", + "revision", + "build", + "model", + "prompt", + "accelerator", + "config", + "comparison", + "environment", + "paths", + "storage_policy", + "authorization", + "audits", + "expected", + } + if self.manifest["runtime"] == "llama.cpp": + top_level.add("candidate") + else: + top_level.update({"host", "oracle"}) + _require_exact_keys(self.manifest, top_level, f"{self.manifest['runtime']} manifest") + if not isinstance(self.manifest["revision"], str) or not self.manifest["revision"]: + raise TraceError("manifest revision is invalid") + if self.manifest["runtime"] == "ds4" and self.manifest["revision"] != DS4_REVISION: + raise TraceError(f"ds4 revision must be {DS4_REVISION}") + if self.manifest["runtime"] == "llama.cpp" and re.fullmatch( + r"[0-9a-f]{40}", self.manifest["revision"]) is None: + raise TraceError("llama.cpp revision must be the exact full Git revision") + if not isinstance(self.manifest["build"], dict): + raise TraceError("manifest build is invalid") + build_keys = ( + { + "number", "info", "compiler", "target", "path", "sha256", + "runtime_profile", "runtime_receipt_sha256", + "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor", + } + if self.manifest["runtime"] == "llama.cpp" + else { + "compiler", "target", "path", "sha256", + "runtime_profile", "runtime_receipt_sha256", + "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor", + } + ) + _require_exact_keys(self.manifest["build"], build_keys, "manifest build") + build_sha256 = self.manifest["build"].get("sha256", "") + if not isinstance(build_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", build_sha256) is None: + raise TraceError("manifest build SHA-256 is invalid") + for key in build_keys - { + "sha256", "number", "runtime_profile", "runtime_receipt_sha256", + "runtime_libraries", "runtime_libraries_post", "runtime_module_monitor"}: + value = self.manifest["build"].get(key) + if not isinstance(value, str) or not value: + raise TraceError(f"manifest build {key} is invalid") + if "number" in build_keys and type(self.manifest["build"]["number"]) is not int: + raise TraceError("manifest build number is invalid") + build_path = self.manifest["build"]["path"] + if not build_path.startswith("/") or ".." in PurePosixPath(build_path).parts or ( + str(PurePosixPath(build_path)) != build_path): + raise TraceError("manifest build path is not canonical") + if self.manifest["runtime"] == "llama.cpp": + libraries = self.manifest["build"]["runtime_libraries"] + if not isinstance(libraries, list) or not libraries: + raise TraceError("manifest runtime library identities are invalid") + post_libraries = self.manifest["build"]["runtime_libraries_post"] + if post_libraries != libraries: + raise TraceError("manifest runtime library closure changed during trace generation") + module_monitor = self.manifest["build"]["runtime_module_monitor"] + if not isinstance(module_monitor, dict): + raise TraceError("manifest runtime module monitor is invalid") + _require_exact_keys( + module_monitor, + {"mechanism", "checked_after_trace", "project_additions"}, + "manifest runtime module monitor", + ) + if module_monitor["mechanism"] not in {"dyld-add-image", "pre-post-snapshot"}: + raise TraceError("manifest runtime module monitor mechanism is invalid") + if module_monitor["checked_after_trace"] is not True: + raise TraceError("manifest runtime module monitor did not complete") + if module_monitor["project_additions"] != []: + raise TraceError("manifest records a runtime module addition during trace generation") + profile = self.manifest["build"]["runtime_profile"] + if not isinstance(profile, dict): + raise TraceError("manifest runtime profile is invalid") + _require_exact_keys( + profile, + {"name", "components", "selected_backend_component"}, + "manifest runtime profile", + ) + profile_name = profile.get("name") + components = profile.get("components") + selected_backend_component = profile.get("selected_backend_component") + if profile_name not in {"co-located", "sibling-lib"}: + raise TraceError("manifest runtime profile name is invalid") + if not isinstance(components, list) or components != sorted(components) or ( + len(components) != len(set(components))) or any( + not isinstance(component, str) or re.fullmatch(r"[a-z0-9-]+", component) is None + for component in components): + raise TraceError("manifest runtime profile components are invalid") + if not {"llama-common", "llama", "ggml", "ggml-base"}.issubset(set(components)): + raise TraceError("manifest runtime profile is missing core components") + if not isinstance(selected_backend_component, str) or ( + selected_backend_component not in components or + selected_backend_component in {"ggml", "ggml-base"} or + not selected_backend_component.startswith("ggml-")): + raise TraceError("manifest selected backend component is invalid") + roles = set() + paths = set() + library_components = set() + previous_path = None + executable_path = PurePosixPath(build_path) + binary_directory = executable_path.parent + library_directory = binary_directory.parent / "lib" + for library in libraries: + _require_exact_keys( + library, + {"component", "filename", "path", "sha256", "role", "revision"}, + "manifest runtime library", + ) + component = library.get("component") + filename = library.get("filename") + path = library.get("path") + digest = library.get("sha256") + role = library.get("role") + revision = library.get("revision") + if component not in components or component in library_components: + raise TraceError("manifest runtime library component is invalid") + library_components.add(component) + if not isinstance(filename, str) or PurePosixPath(filename).name != filename: + raise TraceError("manifest runtime library filename is invalid") + expected_role = { + "llama-common": "build-info", + "llama": "llama", + "ggml-base": "ggml", + selected_backend_component: "selected-backend", + }.get(component, f"runtime:{component}") + if role != expected_role or role in roles: + raise TraceError("manifest runtime library role is invalid") + roles.add(role) + if not isinstance(path, str) or not path.startswith("/") or ".." in PurePosixPath(path).parts or ( + str(PurePosixPath(path)) != path): + raise TraceError("manifest runtime library path is not canonical") + if path in paths: + raise TraceError("manifest runtime library path is duplicated") + if previous_path is not None and path <= previous_path: + raise TraceError("manifest runtime library paths are not sorted") + runtime_path = PurePosixPath(path) + try: + runtime_path.relative_to(library_directory) + in_library_directory = True + except ValueError: + in_library_directory = False + if runtime_path != executable_path and runtime_path.parent != binary_directory and ( + not in_library_directory): + raise TraceError("manifest runtime library is outside the exporter runtime directory") + paths.add(path) + previous_path = path + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError("manifest runtime library SHA-256 is invalid") + if component in {"llama-common", "ggml-base"}: + if revision != self.manifest["revision"]: + raise TraceError("manifest runtime library revision is invalid") + elif revision is not None: + raise TraceError("manifest runtime library revision is unexpected") + expected_directory = binary_directory if profile_name == "co-located" else library_directory + if runtime_path.parent != expected_directory or runtime_path.name != filename: + raise TraceError("manifest runtime library path differs from the runtime profile") + if library_components != set(components): + raise TraceError("manifest runtime library set differs from the runtime profile") + receipt = { + "format": "dsv41-runtime-receipt", + "version": 1, + "revision": self.manifest["revision"], + "profile": profile_name, + "components": sorted( + [ + { + "component": library["component"], + "filename": library["filename"], + "sha256": library["sha256"], + "revision": library["revision"], + } + for library in libraries + ], + key=lambda item: item["component"], + ), + } + receipt_sha256 = self.manifest["build"].get("runtime_receipt_sha256") + if not isinstance(receipt_sha256, str) or receipt_sha256 != sha256_bytes( + canonical_json(receipt).encode("ascii")): + raise TraceError("manifest runtime receipt SHA-256 is invalid") + else: + ds4_policy, _ds4_policy_sha256 = ds4_exporter_approval( + self.verifier.expected_ds4_exporter_policy_id or "", + policies=self.verifier.ds4_exporter_policies, + ) + build_evidence = { + key: self.manifest["build"][key] + for key in ( + "revision", + "path", + "sha256", + "runtime_profile", + "runtime_receipt_sha256", + "runtime_libraries", + "runtime_libraries_post", + ) + if key in self.manifest["build"] + } + build_evidence["revision"] = self.manifest["revision"] + validate_runtime_build_evidence( + build_evidence, + ds4_policy, + label="ds4 exporter", + ) + module_monitor = self.manifest["build"]["runtime_module_monitor"] + if not isinstance(module_monitor, dict): + raise TraceError("ds4 manifest runtime module monitor is invalid") + _require_exact_keys( + module_monitor, + {"mechanism", "checked_after_trace", "project_additions"}, + "ds4 manifest runtime module monitor", + ) + if module_monitor["mechanism"] != "dyld-add-image" or ( + module_monitor["checked_after_trace"] is not True) or ( + module_monitor["project_additions"] != []): + raise TraceError("ds4 manifest runtime module monitor is invalid") + receipt = ds4_policy["runtime_receipt"] + for section in ("model", "prompt"): + if not isinstance(self.manifest[section], dict): + raise TraceError(f"manifest {section} is invalid") + _require_exact_keys( + self.manifest["model"], + {"path", "sha256", "byte_count", "architecture"}, + "manifest model", + ) + _require_exact_keys( + self.manifest["prompt"], + { + "path", + "sha256", + "byte_count", + "corpus_name", + "corpus_sha256", + "target_tokens", + "provenance", + }, + "manifest prompt", + ) + for section in ("model", "prompt"): + digest = self.manifest[section].get("sha256") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError(f"manifest {section} SHA-256 is invalid") + if type(self.manifest[section].get("byte_count")) is not int or self.manifest[section]["byte_count"] <= 0: + raise TraceError(f"manifest {section} byte_count is invalid") + for section in ("config", "comparison", "environment", "audits"): + if not isinstance(self.manifest[section], dict): + raise TraceError(f"manifest {section} is invalid") + _require_exact_keys( + self.manifest["environment"], + {"system_info", "command"}, + "manifest environment", + ) + if not all(isinstance(value, str) and value for value in self.manifest["environment"].values()): + raise TraceError("manifest environment values are invalid") + system_info = self.manifest["environment"]["system_info"].lower() + if self.manifest["runtime"] == "llama.cpp" and "linux" not in system_info: + raise TraceError("llama.cpp environment is not Linux") + if self.manifest["runtime"] == "ds4" and not any( + name in system_info for name in ("darwin", "macos")): + raise TraceError("ds4 environment is not macOS") + context = self.manifest["config"].get("context") + decode_steps = self.manifest["config"].get("decode_steps") + if type(context) is not int or type(decode_steps) is not int or ( + decode_steps <= 0 or context <= decode_steps): + raise TraceError("manifest context or decode_steps is invalid") + if self.manifest["model"]["sha256"] != MODEL_SHA256: + raise TraceError(f"model SHA-256 must be {MODEL_SHA256}") + if self.manifest["model"].get("architecture") != "deepseek41": + raise TraceError("model architecture must be deepseek41") + if self.manifest["storage_policy"] != NO_EXTERNAL_STATE_STORAGE: + raise TraceError("manifest external cache/state storage policy is invalid") + accelerator = validate_accelerator_attestation(self.manifest["runtime"], self.manifest["accelerator"]) + if self.manifest["runtime"] == "ds4": + if "host" not in self.manifest: + raise TraceError("ds4 host attestation is missing") + validate_host_attestation(self.manifest["host"]) + else: + if "host" in self.manifest: + raise TraceError("llama.cpp manifest must not contain Apple host attestation") + required_paths = { + "model", "prompt", "output", "repository", "temporary_directory", + } + if self.manifest["runtime"] == "ds4": + required_paths.update({"runtime_checkout", "runner_executable", "runner_script", "exporter"}) + paths = self.manifest["paths"] + if not isinstance(paths, dict) or set(paths) != required_paths: + raise TraceError("manifest execution paths are invalid") + for label, value in paths.items(): + if not isinstance(label, str) or not isinstance(value, str) or not value.startswith("/"): + raise TraceError("manifest execution path is invalid") + if self.manifest["model"].get("path") != paths["model"]: + raise TraceError("manifest model path is not bound to execution paths") + if self.manifest["prompt"].get("path") != paths["prompt"]: + raise TraceError("manifest prompt path is not bound to execution paths") + if self.manifest["runtime"] == "ds4" and self.manifest["build"]["path"] != paths["exporter"]: + raise TraceError("ds4 build path is not bound to the executed exporter") + corpus_name = self.manifest["prompt"].get("corpus_name") + if corpus_name not in CORPUS_SHA256: + raise TraceError("prompt corpus is not in the fixed correctness corpus set") + if self.manifest["prompt"].get("corpus_sha256") != CORPUS_SHA256[corpus_name]: + raise TraceError(f"prompt corpus SHA-256 is invalid for {corpus_name}") + provenance = self.manifest["prompt"].get("provenance") + if not isinstance(provenance, dict): + raise TraceError("prompt provenance reference is missing") + _require_exact_keys(provenance, {"path", "sha256"}, "prompt provenance reference") + provenance_sha256 = provenance.get("sha256", "") + if not isinstance(provenance_sha256, str) or re.fullmatch( + r"[0-9a-f]{64}", provenance_sha256) is None: + raise TraceError("prompt provenance SHA-256 is invalid") + if provenance.get("path") != f"provenance/{provenance_sha256}.json": + raise TraceError("prompt provenance path is not content addressed") + try: + provenance_bytes = self._read_verified_file(provenance["path"]) + provenance_record = strict_json_loads(provenance_bytes.decode("ascii")) + except (OSError, UnicodeError, TraceError) as error: + raise TraceError(f"cannot read prompt provenance: {error}") from error + if sha256_bytes(provenance_bytes) != provenance_sha256: + raise TraceError("prompt provenance SHA-256 mismatch") + expected_target = context - decode_steps + prompt_policy, prompt_policy_sha256 = prompt_builder_approval( + self.verifier.expected_prompt_builder_policy_id, + policies=self.verifier.prompt_builder_policies, + ) + source_root_lexical = Path(prompt_policy["source_root"]) + source_root_resolved = source_root_lexical.resolve(strict=False) + corpus_resolved = ( + source_root_resolved / "tests" / "corpus" / corpus_name).resolve(strict=False) + provenance_checks = { + "format": "dsv41-prompt-provenance", + "version": 2, + "corpus_name": corpus_name, + "corpus_sha256": self.manifest["prompt"]["corpus_sha256"], + "corpus_path": str(corpus_resolved), + "corpus_resolved_path": str(corpus_resolved), + "source_root_lexical_path": str(source_root_lexical), + "source_root_resolved_path": str(source_root_resolved), + "model_sha256": self.manifest["model"]["sha256"], + "prompt_sha256": self.manifest["prompt"]["sha256"], + "prompt_byte_count": self.manifest["prompt"]["byte_count"], + "context": context, + "decode_steps": decode_steps, + "target_tokens": expected_target, + "actual_tokens": expected_target, + } + for key, value in provenance_checks.items(): + if provenance_record.get(key) != value: + raise TraceError(f"prompt provenance {key} mismatch") + try: + if Path(str(provenance_record.get("corpus_lexical_path"))).resolve(strict=False) != corpus_resolved: + raise TraceError("prompt provenance corpus lexical path resolves outside the approved source") + except OSError as error: + raise TraceError(f"prompt provenance corpus lexical path is invalid: {error}") from error + expected_prompt = approved_prompt_record( + prompt_policy, + corpus_name=corpus_name, + context=context, + decode_steps=decode_steps, + ) + builder_checks = { + "builder_approval_id": self.verifier.expected_prompt_builder_policy_id, + "builder_approval_sha256": prompt_policy_sha256, + "builder_path": prompt_policy["executable_path"], + "builder_sha256": prompt_policy["executable_sha256"], + "builder_revision": prompt_policy["revision"], + "builder_runtime_profile": prompt_policy["runtime_profile"], + "tokenizer": prompt_policy["tokenizer"], + } + for key, value in builder_checks.items(): + if provenance_record.get(key) != value: + raise TraceError(f"prompt provenance {key} differs from external approval") + builder_runtime_build = validate_runtime_build_evidence( + provenance_record.get("builder_runtime_build"), + prompt_policy, + label="prompt builder", + ) + builder_runtime_build_sha256 = runtime_build_evidence_sha256( + builder_runtime_build, + prompt_policy, + label="prompt builder", + ) + if provenance_record.get("builder_runtime_build_sha256") != builder_runtime_build_sha256: + raise TraceError("prompt provenance runtime build SHA-256 mismatch") + builder_trust = validate_install_trust_evidence( + provenance_record.get("builder_install_trust"), prompt_policy) + builder_trust_sha256 = install_trust_sha256(builder_trust) + if provenance_record.get("builder_install_trust_sha256") != builder_trust_sha256: + raise TraceError("prompt provenance install trust SHA-256 mismatch") + if self.manifest["authorization"]["approvals"]["prompt_builder"][ + "install_trust_sha256"] != builder_trust_sha256: + raise TraceError("manifest prompt builder trust differs from signed provenance") + for key in ("corpus_name", "corpus_sha256", "context", "decode_steps", "target_tokens", + "prompt_sha256", "prompt_byte_count"): + if provenance_record[key] != expected_prompt[key]: + raise TraceError(f"prompt provenance {key} differs from approved prompt output") + _require_exact_keys( + provenance_record, + set(provenance_checks) | set(builder_checks) | { + "corpus_lexical_path", + "builder_runtime_build", "builder_runtime_build_sha256", + "builder_install_trust", "builder_install_trust_sha256"}, + "prompt provenance", + ) + if self.manifest["prompt"].get("target_tokens") != expected_target: + raise TraceError("prompt target token count does not fill the configured context") + if self.manifest["runtime"] == "llama.cpp": + candidate = self.manifest.get("candidate") + if not isinstance(candidate, dict): + raise TraceError("llama.cpp candidate attestation is missing") + _require_exact_keys( + candidate, + { + "repository", + "revision", + "base_revision", + "diff_sha256", + "executable_path", + "executable_sha256", + "runtime_libraries_sha256", + "runtime_receipt_sha256", + "exporter_approval_id", + "exporter_approval_sha256", + "install_trust", + "install_trust_sha256", + }, + "llama.cpp candidate attestation", + ) + if candidate.get("repository") != REPOSITORY: + raise TraceError(f"candidate repository must be {REPOSITORY}") + for key in ( + "revision", + "base_revision", + "diff_sha256", + "executable_sha256", + "runtime_libraries_sha256", + "runtime_receipt_sha256", + "exporter_approval_sha256"): + value = candidate.get(key, "") + if not isinstance(value, str) or re.fullmatch( + r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: + raise TraceError(f"candidate {key} is invalid") + if re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", candidate.get("exporter_approval_id", "")) is None: + raise TraceError("candidate exporter approval ID is invalid") + executable_path = candidate.get("executable_path") + if not isinstance(executable_path, str) or not executable_path.startswith("/") or ( + ".." in PurePosixPath(executable_path).parts or + str(PurePosixPath(executable_path)) != executable_path): + raise TraceError("candidate executable path is invalid") + if candidate["revision"] != self.manifest["revision"]: + raise TraceError("candidate revision does not match the exporter build revision") + if candidate.get("executable_path") != self.manifest["build"]["path"]: + raise TraceError("candidate executable path does not match the trace build") + if candidate["executable_sha256"] != self.manifest["build"]["sha256"]: + raise TraceError("candidate executable SHA-256 does not match the trace build") + runtime_libraries_sha256 = sha256_bytes( + canonical_json({ + "pre": self.manifest["build"]["runtime_libraries"], + "post": self.manifest["build"]["runtime_libraries_post"], + }).encode("ascii")) + if candidate["runtime_libraries_sha256"] != runtime_libraries_sha256: + raise TraceError("candidate runtime library identities do not match the trace build") + if candidate["runtime_receipt_sha256"] != self.manifest["build"]["runtime_receipt_sha256"]: + raise TraceError("candidate runtime receipt does not match the trace build") + candidate_policy, candidate_policy_sha256 = candidate_exporter_approval( + self.verifier.expected_candidate_exporter_policy_id or "", + policies=self.verifier.candidate_exporter_policies, + ) + if candidate["exporter_approval_id"] != self.verifier.expected_candidate_exporter_policy_id or ( + candidate["exporter_approval_sha256"] != candidate_policy_sha256): + raise TraceError("candidate exporter approval differs from external policy") + policy_checks = { + "repository": candidate["repository"], + "revision": candidate["revision"], + "base_revision": candidate["base_revision"], + "diff_sha256": candidate["diff_sha256"], + "executable_path": candidate["executable_path"], + "executable_sha256": candidate["executable_sha256"], + "runtime_profile": self.manifest["build"]["runtime_profile"], + "runtime_receipt": receipt, + } + for key, value in policy_checks.items(): + if candidate_policy[key] != value: + raise TraceError(f"candidate {key} differs from external exporter approval") + candidate_trust = validate_install_trust_evidence( + candidate["install_trust"], candidate_policy) + candidate_trust_sha256 = install_trust_sha256(candidate_trust) + if candidate["install_trust_sha256"] != candidate_trust_sha256 or ( + self.manifest["authorization"]["approvals"]["candidate_exporter"][ + "install_trust_sha256"] != candidate_trust_sha256): + raise TraceError("candidate install trust evidence is not bound to authorization") + else: + oracle = self.manifest.get("oracle") + if not isinstance(oracle, dict): + raise TraceError("ds4 oracle attestation is missing") + _require_exact_keys( + oracle, + { + "repository", + "revision", + "verifier_revision", + "executable_path", + "executable_sha256", + "runtime_profile", + "runtime_build_sha256", + "runtime_libraries_sha256", + "runtime_receipt_sha256", + "exporter_approval_id", + "exporter_approval_sha256", + "install_trust", + "install_trust_sha256", + }, + "ds4 oracle attestation", + ) + for key in ( + "revision", + "verifier_revision", + "executable_sha256", + "runtime_libraries_sha256", + "runtime_build_sha256", + "runtime_receipt_sha256", + "exporter_approval_sha256", + "install_trust_sha256"): + value = oracle.get(key, "") + if not isinstance(value, str) or re.fullmatch( + r"[0-9a-f]{40}" if "revision" in key else r"[0-9a-f]{64}", value) is None: + raise TraceError(f"ds4 oracle {key} is invalid") + if oracle["repository"] != DS4_REPOSITORY or oracle["revision"] != DS4_REVISION or ( + oracle["revision"] == oracle["verifier_revision"]): + raise TraceError("ds4 oracle producer/verifier identity is invalid") + if re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", oracle.get("exporter_approval_id", "")) is None: + raise TraceError("ds4 exporter approval ID is invalid") + if oracle["executable_path"] != self.manifest["build"]["path"] or ( + oracle["executable_sha256"] != self.manifest["build"]["sha256"]): + raise TraceError("ds4 oracle executable identity does not match the trace build") + if oracle["runtime_profile"] != self.manifest["build"]["runtime_profile"]: + raise TraceError("ds4 oracle runtime profile does not match the trace build") + ds4_policy, ds4_policy_sha256 = ds4_exporter_approval( + self.verifier.expected_ds4_exporter_policy_id or "", + policies=self.verifier.ds4_exporter_policies, + ) + build_evidence = { + "revision": self.manifest["revision"], + "path": self.manifest["build"]["path"], + "sha256": self.manifest["build"]["sha256"], + "runtime_profile": self.manifest["build"]["runtime_profile"], + "runtime_receipt_sha256": self.manifest["build"]["runtime_receipt_sha256"], + "runtime_libraries": self.manifest["build"]["runtime_libraries"], + "runtime_libraries_post": self.manifest["build"]["runtime_libraries_post"], + } + if oracle["runtime_build_sha256"] != runtime_build_evidence_sha256( + build_evidence, ds4_policy, label="ds4 exporter"): + raise TraceError("ds4 oracle runtime build SHA-256 does not match the trace build") + runtime_libraries_sha256 = sha256_bytes( + canonical_json({ + "pre": self.manifest["build"]["runtime_libraries"], + "post": self.manifest["build"]["runtime_libraries_post"], + }).encode("ascii")) + if oracle["runtime_libraries_sha256"] != runtime_libraries_sha256 or ( + oracle["runtime_receipt_sha256"] != self.manifest["build"]["runtime_receipt_sha256"]): + raise TraceError("ds4 oracle runtime evidence does not match the trace build") + if oracle["exporter_approval_id"] != self.verifier.expected_ds4_exporter_policy_id or ( + oracle["exporter_approval_sha256"] != ds4_policy_sha256): + raise TraceError("ds4 exporter approval differs from external policy") + policy_checks = { + "repository": oracle["repository"], + "revision": oracle["revision"], + "executable_path": oracle["executable_path"], + "executable_sha256": oracle["executable_sha256"], + "runtime_profile": oracle["runtime_profile"], + "runtime_receipt": receipt, + } + for key, value in policy_checks.items(): + if ds4_policy[key] != value: + raise TraceError(f"ds4 oracle {key} differs from external exporter approval") + if oracle["verifier_revision"] != self.verifier.expected_verifier_revision: + raise TraceError("ds4 oracle verifier revision differs from external approval") + oracle_trust = validate_install_trust_evidence( + oracle["install_trust"], ds4_policy) + oracle_trust_sha256 = install_trust_sha256(oracle_trust) + if oracle["install_trust_sha256"] != oracle_trust_sha256 or ( + self.manifest["authorization"]["approvals"]["ds4_exporter"][ + "install_trust_sha256"] != oracle_trust_sha256): + raise TraceError("ds4 exporter install trust evidence is not bound to authorization") + expected_config = { + "layer_count": 40, + "vocab_size": 129280, + "engram_layers": [1, 14], + "engram_rows_per_token": 24, + "expert_count": 384, + "experts_used": 6, + "candidate_source_layer": 20, + "candidate_topk_blocks": 2048, + "candidate_block_size": 8, + "index_top_k": 512, + "raw_attention_layers": list(RAW_ATTENTION_LAYERS), + "raw_attention_width": RAW_ATTENTION_WIDTH, + "candidate_propagation_layers": [24, 28, 32, 36], + } + if self.manifest["config"].get("deepseek41") != expected_config: + raise TraceError("DeepSeek V4.1 configuration is invalid") + expected_comparison = { + "tokens": "exact", + "engram_rows": "exact", + "expert_ids": "exact-original-id-space", + "expert_weights": "byte-identical-f32", + "attention_candidates": "exact", + "logits": "byte-identical-f32", + } + if self.manifest["comparison"] != expected_comparison: + raise TraceError("trace comparison policy is invalid") + config = self.manifest["config"] + if self.manifest["runtime"] == "llama.cpp": + _require_exact_keys( + config, + { + "context", + "batch", + "ubatch", + "device", + "device_architecture", + "device_pci_id", + "decode_steps", + "kv_type_k", + "kv_type_v", + "flash_attention", + "gpu_layers", + "load_mode", + "expert_cache_slots", + "expert_cache_bytes", + "tokenizer", + "model_file_identity", + "watchdog_namespace", + "deepseek41", + }, + "llama.cpp config", + ) + if accelerator["backend_device"] != "ROCm0": + raise TraceError("llama.cpp accelerator backend device must be ROCm0") + if config.get("batch") != ADMITTED_BATCH or config.get("ubatch") != ADMITTED_UBATCH: + raise TraceError("llama.cpp trace does not use the admitted batch and ubatch") + if config.get("expert_cache_slots") != REQUIRED_EXPERT_SLOTS or ( + config.get("expert_cache_bytes") != REQUIRED_EXPERT_CACHE_BYTES): + raise TraceError("llama.cpp trace does not use the admitted expert cache") + if config.get("device") != "ROCm0" or config.get("gpu_layers") != 99: + raise TraceError("llama.cpp trace does not use the required ROCm0 offload") + if config.get("device_architecture") != accelerator["architecture"] or ( + config.get("device_pci_id") != accelerator["pci_device_id"]): + raise TraceError("llama.cpp trace device identity is not bound to the accelerator attestation") + if config.get("kv_type_k") != "f16" or config.get("kv_type_v") != "f16" or ( + config.get("flash_attention") is not True) or config.get("load_mode") != 0: + raise TraceError("llama.cpp trace inference configuration is invalid") + if validate_tokenizer_policy(config.get("tokenizer")) != prompt_policy["tokenizer"]: + raise TraceError("llama.cpp tokenizer policy differs from external prompt approval") + model_file_identity = config.get("model_file_identity") + if not isinstance(model_file_identity, dict): + raise TraceError("llama.cpp model file identity is invalid") + _require_exact_keys( + model_file_identity, + { + "format", + "version", + "path", + "device", + "inode", + "owner_uid", + "owner_gid", + "mode", + "link_count", + "byte_count", + "modified_ns", + "changed_ns", + "status_flags", + "source_descriptor_flags", + "target_descriptor_flags", + "sha256", + }, + "llama.cpp model file identity", + ) + if model_file_identity["format"] != "dsv41-model-file-identity" or ( + model_file_identity["version"] != 1) or ( + model_file_identity["path"] != self.manifest["model"]["path"]) or ( + model_file_identity["byte_count"] != self.manifest["model"]["byte_count"]) or ( + model_file_identity["sha256"] != self.manifest["model"]["sha256"]): + raise TraceError("llama.cpp model file identity does not match the manifest model") + for key in ( + "device", "inode", "owner_uid", "owner_gid", "mode", "link_count", + "byte_count", "modified_ns", "changed_ns", "status_flags", + "source_descriptor_flags", "target_descriptor_flags"): + if type(model_file_identity[key]) is not int or model_file_identity[key] < 0: + raise TraceError(f"llama.cpp model file identity {key} is invalid") + if model_file_identity["link_count"] < 1 or model_file_identity["mode"] > 0o7777 or ( + model_file_identity["source_descriptor_flags"] != 1) or ( + model_file_identity["target_descriptor_flags"] != 0): + raise TraceError("llama.cpp model descriptor policy is invalid") + watchdog_namespace = config.get("watchdog_namespace") + if not isinstance(watchdog_namespace, dict): + raise TraceError("llama.cpp watchdog namespace binding is invalid") + _require_exact_keys( + watchdog_namespace, + { + "format", + "version", + "authority", + "host_watchdog_pid", + "host_watchdog_process_group_id", + "host_watchdog_start_time_ticks", + "local_pid", + "local_parent_pid", + "local_process_group_id", + "local_session_id", + "namespace_pids", + "private_procfs", + }, + "llama.cpp watchdog namespace binding", + ) + if watchdog_namespace["format"] != "dsv41-watchdog-namespace-binding" or ( + watchdog_namespace["version"] != 1) or ( + watchdog_namespace["authority"] != "inherited-pidfd") or ( + watchdog_namespace["private_procfs"] is not True): + raise TraceError("llama.cpp watchdog namespace binding policy is invalid") + for key in ( + "host_watchdog_pid", "host_watchdog_process_group_id", + "host_watchdog_start_time_ticks", "local_pid", + "local_process_group_id", "local_session_id"): + if type(watchdog_namespace[key]) is not int or watchdog_namespace[key] <= 1: + raise TraceError(f"llama.cpp watchdog namespace {key} is invalid") + if watchdog_namespace["local_parent_pid"] != 1 or ( + watchdog_namespace["local_pid"] != 2) or ( + watchdog_namespace["local_pid"] != watchdog_namespace["local_process_group_id"]) or ( + watchdog_namespace["local_pid"] != watchdog_namespace["local_session_id"]) or ( + not isinstance(watchdog_namespace["namespace_pids"], list)) or ( + not watchdog_namespace["namespace_pids"]) or ( + watchdog_namespace["namespace_pids"][-1] != watchdog_namespace["local_pid"]): + raise TraceError("llama.cpp watchdog namespace-local identity is invalid") + if self.manifest["runtime"] == "ds4": + _require_exact_keys( + config, + { + "context", + "decode_steps", + "prefill_chunk", + "device_backend", + "device_registry_id", + "deepseek41", + }, + "ds4 config", + ) + if config.get("prefill_chunk") != ADMITTED_UBATCH: + raise TraceError("ds4 trace does not use the admitted prefill chunk") + if config.get("device_backend") != "Metal" or ( + config.get("device_registry_id") != accelerator["metal_registry_id"]): + raise TraceError("ds4 trace device identity is not bound to the accelerator attestation") + _require_exact_keys(self.manifest["audits"], {"pre", "post"}, "manifest audit envelope") + for audit_phase in ("pre", "post"): + phase_audits = self.manifest["audits"].get(audit_phase) + if not isinstance(phase_audits, dict): + raise TraceError(f"manifest {audit_phase} audit set is invalid") + expected_kinds = ( + ("memory", "swap", "watchdog") + if self.manifest["runtime"] == "llama.cpp" + else ("memory", "swap", "runner") + ) + if set(phase_audits) != set(expected_kinds): + raise TraceError(f"manifest {audit_phase} audit kinds are invalid") + for kind in expected_kinds: + self._validate_audit_reference(audit_phase, kind, phase_audits.get(kind)) + + def _validate_audit_reference(self, phase: str, kind: str, audit: Any) -> None: + if not isinstance(audit, dict): + raise TraceError(f"manifest {phase} {kind} audit reference is invalid") + _require_exact_keys( + audit, + {"path", "sha256", "created_unix"}, + f"manifest {phase} {kind} audit reference", + ) + audit_path = audit.get("path") + if not isinstance(audit_path, str) or not audit_path: + raise TraceError(f"manifest {phase} {kind} audit path is invalid") + digest = audit.get("sha256", "") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise TraceError(f"manifest {phase} {kind} audit SHA-256 is invalid") + if type(audit.get("created_unix")) is not int or audit["created_unix"] <= 0: + raise TraceError(f"manifest {phase} {kind} audit timestamp is invalid") + expected_path = f"audits/{phase}/{digest}.json" + if audit_path != expected_path: + raise TraceError(f"manifest {phase} {kind} audit path is not content addressed") + try: + evidence = self._read_verified_file(audit_path) + except TraceError as error: + raise TraceError(f"cannot read {phase} {kind} audit evidence: {error}") from error + if sha256_bytes(evidence) != digest: + raise TraceError(f"{phase} {kind} audit evidence SHA-256 mismatch") + try: + record = strict_json_loads(evidence.decode("ascii")) + except (UnicodeError, TraceError) as error: + raise TraceError(f"{phase} {kind} audit evidence is invalid: {error}") from error + if record.get("kind") != kind or record.get("created_unix") != audit["created_unix"]: + raise TraceError(f"{phase} {kind} audit evidence metadata mismatch") + record_keys = {"created_unix", "kind", "environment", "data"} + if kind == "memory": + record_keys |= {"storage", "storage_policy", "accelerator"} + if self.manifest["runtime"] == "ds4": + record_keys.add("host") + _require_exact_keys(record, record_keys, f"{phase} {kind} audit evidence") + expected_environment = ( + {"HIP_LAUNCH_BLOCKING": "1"} + if self.manifest["runtime"] == "llama.cpp" + else {} + ) + if record.get("environment") != expected_environment: + raise TraceError(f"{phase} {kind} audit environment is invalid") + if not isinstance(record.get("data"), dict): + raise TraceError(f"{phase} {kind} audit evidence data is invalid") + if kind == "memory": + _require_exact_keys( + record["data"], + {"mem_total_bytes", "mem_available_bytes", "mem_used_bytes"}, + f"{phase} memory audit data", + ) + used = record["data"].get("mem_used_bytes") + total = record["data"].get("mem_total_bytes") + available = record["data"].get("mem_available_bytes") + if type(used) is not int or used < 0: + raise TraceError(f"{phase} memory audit evidence is invalid") + if self.manifest["runtime"] == "llama.cpp" and used >= SOFT_MEMORY_LIMIT: + raise TraceError(f"{phase} memory audit evidence is invalid") + if self.manifest["runtime"] == "ds4" and ( + type(total) is not int or total < 128 * 1024 * 1024 * 1024 or + type(available) is not int or available <= 0 or available > total or used > total): + raise TraceError(f"{phase} ds4 memory audit evidence is invalid") + storage = record.get("storage") + if record.get("storage_policy") != self.manifest["storage_policy"]: + raise TraceError(f"{phase} memory audit storage policy mismatch") + required_storage = { + "model", "prompt", "output", "repository", "temporary_directory", + } + if self.manifest["runtime"] == "ds4": + required_storage.update({"runtime_checkout", "runner_executable", "runner_script", "exporter"}) + if not isinstance(storage, dict) or set(storage) != required_storage: + raise TraceError(f"{phase} memory audit storage evidence is missing") + for label, item in storage.items(): + if not isinstance(label, str): + raise TraceError(f"{phase} memory audit storage evidence is invalid") + validate_storage_attestation(self.manifest["runtime"], item) + if item["resolved_path"] != self.manifest["paths"][label]: + raise TraceError(f"{phase} memory audit {label} path differs from the manifest") + if record.get("accelerator") != self.manifest["accelerator"]: + raise TraceError(f"{phase} memory audit accelerator evidence mismatch") + if self.manifest["runtime"] == "ds4": + host = validate_host_attestation(record.get("host")) + if host != self.manifest["host"]: + raise TraceError(f"{phase} ds4 host evidence mismatch") + if host["memory_bytes"] != total: + raise TraceError(f"{phase} ds4 host memory differs from the memory audit") + if kind == "swap": + if self.manifest["runtime"] == "llama.cpp": + _require_exact_keys(record["data"], {"enabled", "entries"}, f"{phase} swap audit data") + if record["data"].get("enabled") is not False or record["data"].get("entries") != []: + raise TraceError(f"{phase} swap audit evidence does not report zero configured swap") + elif record["data"] != { + "source": "darwin-sysctl-vm.swapusage", + "total_bytes": 0, + "used_bytes": 0, + "free_bytes": 0, + }: + raise TraceError(f"{phase} ds4 swap audit evidence does not report zero swap") + if kind == "runner": + data = record["data"] + _require_exact_keys( + data, + { + "format", + "version", + "runtime_kind", + "source", + "runner_pid", + "runner_parent_pid", + "runner_uid", + "runner_executable", + "runner_executable_sha256", + "runner_script", + "runner_script_sha256", + "exporter_path", + "exporter_sha256", + "exporter_approval_id", + "exporter_approval_sha256", + "exporter_install_trust_sha256", + "exporter_runtime_build_sha256", + "exporter_runtime_profile", + "exporter_runtime_receipt_sha256", + "producer_revision", + "verifier_revision", + "checkout_path", + "checkout_revision", + "command_sha256", + }, + f"{phase} ds4 runner audit", + ) + expected = { + "format": "dsv41-runner-ownership", + "version": 1, + "runtime_kind": "apple-metal", + "source": "python-subprocess", + "exporter_sha256": self.manifest["build"]["sha256"], + "exporter_approval_id": self.manifest["oracle"]["exporter_approval_id"], + "exporter_approval_sha256": self.manifest["oracle"]["exporter_approval_sha256"], + "exporter_install_trust_sha256": self.manifest["oracle"]["install_trust_sha256"], + "exporter_runtime_build_sha256": self.manifest["oracle"]["runtime_build_sha256"], + "exporter_runtime_profile": self.manifest["oracle"]["runtime_profile"], + "exporter_runtime_receipt_sha256": self.manifest["oracle"]["runtime_receipt_sha256"], + "producer_revision": self.manifest["oracle"]["revision"], + "verifier_revision": self.manifest["oracle"]["verifier_revision"], + "checkout_revision": DS4_REVISION, + "checkout_path": self.manifest["paths"]["runtime_checkout"], + "runner_executable": self.manifest["paths"]["runner_executable"], + "runner_script": self.manifest["paths"]["runner_script"], + "exporter_path": self.manifest["paths"]["exporter"], + } + for key, value in expected.items(): + if data.get(key) != value: + raise TraceError(f"{phase} ds4 runner {key} mismatch") + for key in ("runner_pid", "runner_parent_pid"): + if type(data.get(key)) is not int or data[key] <= 0: + raise TraceError(f"{phase} ds4 runner {key} is invalid") + if type(data.get("runner_uid")) is not int or data["runner_uid"] < 0: + raise TraceError(f"{phase} ds4 runner UID is invalid") + for key in ( + "runner_executable_sha256", + "runner_script_sha256", + "exporter_sha256", + "exporter_approval_sha256", + "exporter_install_trust_sha256", + "exporter_runtime_build_sha256", + "exporter_runtime_receipt_sha256", + "command_sha256"): + if re.fullmatch(r"[0-9a-f]{64}", data.get(key, "")) is None: + raise TraceError(f"{phase} ds4 runner {key} is invalid") + if re.fullmatch( + r"[A-Za-z0-9._-]{1,128}", data.get("exporter_approval_id", "")) is None: + raise TraceError(f"{phase} ds4 runner exporter approval ID is invalid") + for key in ("producer_revision", "verifier_revision"): + if re.fullmatch(r"[0-9a-f]{40}", data.get(key, "")) is None: + raise TraceError(f"{phase} ds4 runner {key} is invalid") + if not isinstance(data.get("exporter_runtime_profile"), dict): + raise TraceError(f"{phase} ds4 runner runtime profile is invalid") + if kind == "watchdog": + required = ( + "format", + "version", + "lease_id", + "state", + "file_device", + "file_inode", + "file_uid", + "file_mode", + "lease_path", + "watchdog_pid", + "watchdog_start_time_utc", + "watchdog_start_time_ticks", + "watchdog_command", + "watchdog_command_sha256", + "watchdog_executable_path", + "watchdog_script_path", + "watchdog_script_sha256", + "watchdog_revision", + "soft_bytes", + "emergency_bytes", + "strict_ceiling_bytes", + "grace_seconds", + "sample_interval_seconds", + "procfs_root", + "guardian_pid", + "child_pid", + "child_process_group_id", + "command", + "child_command_sha256", + "heartbeat_path", + "heartbeat_unix", + "max_heartbeat_age_seconds", + "audit_live_path", + "audit_device", + "audit_inode", + "audit_uid", + "audit_mode", + "audit_fd", + "audit_sha256", + "audit", + "namespace_authority", + ) + _require_exact_keys(record["data"], set(required), f"{phase} watchdog audit evidence") + data = record["data"] + if data["format"] != WATCHDOG_LEASE_FORMAT or data["version"] != WATCHDOG_VERSION: + raise TraceError(f"{phase} watchdog audit format is invalid") + if data["state"] != "active": + raise TraceError(f"{phase} watchdog audit state is invalid") + for key in ("file_device", "file_inode", "file_uid"): + if type(data[key]) is not int or data[key] < 0: + raise TraceError(f"{phase} watchdog lease {key} is invalid") + if data["file_mode"] != 0o600: + raise TraceError(f"{phase} watchdog lease file mode is invalid") + if not isinstance(data["lease_id"], str) or re.fullmatch(r"[0-9a-f]{32,64}", data["lease_id"]) is None: + raise TraceError(f"{phase} watchdog audit lease ID is invalid") + if type(data["watchdog_pid"]) is not int or data["watchdog_pid"] <= 1: + raise TraceError(f"{phase} watchdog audit PID is invalid") + if type(data["watchdog_start_time_ticks"]) is not int or data["watchdog_start_time_ticks"] <= 0: + raise TraceError(f"{phase} watchdog audit start time is invalid") + if not isinstance(data["watchdog_start_time_utc"], str) or re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z", + data["watchdog_start_time_utc"]) is None: + raise TraceError(f"{phase} watchdog audit UTC start time is invalid") + for key in ("watchdog_command_sha256", "watchdog_script_sha256"): + if not isinstance(data[key], str) or re.fullmatch(r"[0-9a-f]{64}", data[key]) is None: + raise TraceError(f"{phase} watchdog audit {key} is invalid") + for key in ("lease_path", "watchdog_command", "watchdog_script_path", "heartbeat_path", "audit_live_path"): + if not isinstance(data[key], str) or not data[key]: + raise TraceError(f"{phase} watchdog audit {key} is invalid") + if APPROVED_WATCHDOGS.get(data["watchdog_script_sha256"]) != data["watchdog_revision"]: + raise TraceError(f"{phase} watchdog audit revision is invalid") + if not isinstance(data["watchdog_executable_path"], str) or not data["watchdog_executable_path"]: + raise TraceError(f"{phase} watchdog executable path is invalid") + if data["soft_bytes"] != SOFT_MEMORY_LIMIT or ( + data["emergency_bytes"] != WATCHDOG_EMERGENCY_LIMIT) or ( + data["strict_ceiling_bytes"] != STRICT_MEMORY_LIMIT): + raise TraceError(f"{phase} watchdog audit thresholds are invalid") + if data["grace_seconds"] != 30.0 or data["sample_interval_seconds"] != 1.0: + raise TraceError(f"{phase} watchdog timing policy is invalid") + if data["procfs_root"] != "/proc": + raise TraceError(f"{phase} watchdog procfs root is invalid") + if type(data["guardian_pid"]) is not int or data["guardian_pid"] <= 1 or ( + type(data["child_pid"]) is not int or data["child_pid"] <= 1) or ( + type(data["child_process_group_id"]) is not int or data["child_process_group_id"] <= 1): + raise TraceError(f"{phase} watchdog child identity is invalid") + for key in ("audit_device", "audit_inode", "audit_uid", "audit_fd"): + if type(data[key]) is not int or data[key] < 0: + raise TraceError(f"{phase} watchdog audit {key} is invalid") + if data["audit_mode"] != 0o600: + raise TraceError(f"{phase} watchdog audit mode is invalid") + if not isinstance(data["command"], list) or not data["command"] or ( + not all(isinstance(argument, str) for argument in data["command"])): + raise TraceError(f"{phase} watchdog child command is invalid") + canonical_command = json.dumps( + data["command"], ensure_ascii=True, separators=(",", ":")).encode("utf-8") + if data["child_command_sha256"] != sha256_bytes(canonical_command): + raise TraceError(f"{phase} watchdog child command SHA-256 is invalid") + if not isinstance(data["watchdog_command_sha256"], str) or re.fullmatch( + r"[0-9a-f]{64}", data["watchdog_command_sha256"]) is None: + raise TraceError(f"{phase} watchdog audit command SHA-256 is invalid") + if type(data["heartbeat_unix"]) is not int or data["heartbeat_unix"] <= 0: + raise TraceError(f"{phase} watchdog audit heartbeat timestamp is invalid") + namespace_authority = data["namespace_authority"] + if not isinstance(namespace_authority, dict): + raise TraceError(f"{phase} watchdog namespace authority is invalid") + _require_exact_keys( + namespace_authority, + { + "format", + "version", + "mechanism", + "descriptor", + "host_procfs_root", + "watchdog_pid", + "watchdog_process_group_id", + "watchdog_start_time_ticks", + "watchdog_executable_path", + "watchdog_command_sha256", + "guardian_pid", + "child_pid", + "child_process_group_id", + }, + f"{phase} watchdog namespace authority", + ) + if namespace_authority["format"] != "dsv41-watchdog-namespace-authority" or ( + namespace_authority["version"] != 1) or ( + namespace_authority["mechanism"] != "inherited-pidfd") or ( + namespace_authority["host_procfs_root"] != "/proc"): + raise TraceError(f"{phase} watchdog namespace authority policy is invalid") + authority_checks = { + "watchdog_pid": data["watchdog_pid"], + "watchdog_start_time_ticks": data["watchdog_start_time_ticks"], + "watchdog_executable_path": data["watchdog_executable_path"], + "watchdog_command_sha256": data["watchdog_command_sha256"], + "guardian_pid": data["guardian_pid"], + "child_pid": data["child_pid"], + "child_process_group_id": data["child_process_group_id"], + } + for key, value in authority_checks.items(): + if namespace_authority.get(key) != value: + raise TraceError(f"{phase} watchdog namespace authority {key} mismatch") + for key in ("descriptor", "watchdog_process_group_id"): + if type(namespace_authority[key]) is not int or namespace_authority[key] <= 2: + raise TraceError(f"{phase} watchdog namespace authority {key} is invalid") + max_age = data.get("max_heartbeat_age_seconds") + if not isinstance(max_age, (int, float)) or isinstance(max_age, bool) or ( + max_age <= 0 or max_age > 30): + raise TraceError(f"{phase} watchdog audit heartbeat age is invalid") + if data["heartbeat_unix"] > record["created_unix"] or ( + record["created_unix"] - data["heartbeat_unix"] > max_age): + raise TraceError(f"{phase} watchdog audit heartbeat was stale when captured") + audit_jsonl = data["audit"] + if not isinstance(audit_jsonl, dict): + raise TraceError(f"{phase} watchdog JSONL reference is invalid") + _require_exact_keys( + audit_jsonl, + {"path", "sha256", "event_count"}, + f"{phase} watchdog JSONL reference", + ) + jsonl_digest = audit_jsonl.get("sha256", "") + if not isinstance(jsonl_digest, str) or re.fullmatch(r"[0-9a-f]{64}", jsonl_digest) is None: + raise TraceError(f"{phase} watchdog JSONL SHA-256 is invalid") + if data["audit_sha256"] != jsonl_digest: + raise TraceError(f"{phase} watchdog live and embedded audit SHA-256 differ") + if audit_jsonl.get("path") != f"audits/{phase}/{jsonl_digest}.jsonl": + raise TraceError(f"{phase} watchdog JSONL path is invalid") + if type(audit_jsonl.get("event_count")) is not int or audit_jsonl["event_count"] < 2: + raise TraceError(f"{phase} watchdog JSONL event count is invalid") + try: + jsonl_bytes = self._read_verified_file(audit_jsonl["path"]) + except TraceError as error: + raise TraceError(f"cannot read {phase} watchdog JSONL audit: {error}") from error + if sha256_bytes(jsonl_bytes) != jsonl_digest: + raise TraceError(f"{phase} watchdog JSONL SHA-256 mismatch") + lines = jsonl_bytes.decode("ascii").splitlines() + if len(lines) != audit_jsonl["event_count"]: + raise TraceError(f"{phase} watchdog JSONL event count mismatch") + try: + events = [validate_watchdog_event(strict_json_loads(line)) for line in lines] + except TraceError as error: + raise TraceError(f"{phase} watchdog JSONL is invalid: {error}") from error + if not any(event.get("event") == "preflight" for event in events) or ( + not any(event.get("event") == "child_started" for event in events)): + raise TraceError(f"{phase} watchdog JSONL lacks startup evidence") + + def read_blob(self, event: dict[str, Any]) -> bytes: + return self._read_verified_file(event["blob"]) + + def _read_verified_file(self, relative: str) -> bytes: + retained = self._retained_files.get(relative) + if retained is not None: + return retained + expected = self._file_receipts.get(relative) + if expected is None: + raise TraceError(f"trace file is outside the signed domain: {relative}") + receipt, data = _read_bundle_file(self.root, relative, retain=True) + if receipt != expected: + raise TraceError(f"trace bundle file changed after signature verification: {relative}") + assert data is not None + return data + + def _path(self, relative: str) -> Path: + parts = _bundle_path_parts(relative) + candidate = self.root + for part in parts: + candidate = candidate / part + if candidate.is_symlink(): + raise TraceError(f"trace path must not use symlinks: {relative}") + try: + candidate.resolve().relative_to(self.root) + except ValueError as error: + raise TraceError(f"trace path is outside the bundle: {relative}") from error + return candidate + + def _validate_coverage(self) -> None: + expected = self.manifest.get("expected") + if not isinstance(expected, dict): + raise TraceError("manifest expected coverage is missing") + _require_exact_keys( + expected, + {"prompt_tokens", "decode_steps", "components"}, + "manifest expected coverage", + ) + prompt_tokens = expected.get("prompt_tokens") + decode_steps = expected.get("decode_steps") + components = expected.get("components") + if type(prompt_tokens) is not int or prompt_tokens <= 0: + raise TraceError("expected prompt_tokens is invalid") + if type(decode_steps) is not int or decode_steps <= 0: + raise TraceError("expected decode_steps is invalid") + if self.manifest.get("config", {}).get("decode_steps") != decode_steps: + raise TraceError("expected decode_steps does not match config") + if prompt_tokens != self.manifest.get("prompt", {}).get("target_tokens"): + raise TraceError("expected prompt_tokens does not match prompt provenance") + for event in self.events: + self._validate_component_schema(event) + raw_attention = { + (event["phase"], event["step"], event["token_start"], event["layer"]): event + for event in self.events + if event["component"] == "attn.source" and event["layer"] in RAW_ATTENTION_LAYERS + } + raw_coordinates = { + (phase, step, token_start) + for phase, step, token_start, _layer in raw_attention + } + for phase, step, token_start in raw_coordinates: + layer0 = raw_attention.get((phase, step, token_start, 0)) + layer1 = raw_attention.get((phase, step, token_start, 1)) + if layer0 is None or layer1 is None: + continue + if layer0["shape"] != layer1["shape"] or self.read_blob(layer0) != self.read_blob(layer1): + raise TraceError( + f"raw attn.source differs between layers 0 and 1 at " + f"{phase} step {step} token {token_start}") + if not isinstance(components, dict): + raise TraceError("expected components are invalid") + if self.manifest.get("model", {}).get("architecture") == "deepseek41": + if components != DEEPSEEK41_EXPECTED_COMPONENTS: + raise TraceError("DeepSeek V4.1 expected component coverage contract is invalid") + + by_component: dict[str, list[dict[str, Any]]] = {} + for event in self.events: + by_component.setdefault(event["component"], []).append(event) + for component in HARD_FAILURE_COMPONENTS: + if component not in components: + raise TraceError(f"expected coverage lacks {component}") + rules = components[component] + events = by_component.get(component, []) + if not events: + raise TraceError(f"trace lacks required component: {component}") + expected_layers = rules.get("layers") + if expected_layers is not None: + actual_layers = sorted({event["layer"] for event in events}) + if actual_layers != expected_layers: + raise TraceError( + f"{component} layer coverage mismatch: expected {expected_layers}, found {actual_layers}") + input_coverage = rules.get("input") + prefill_coverage = rules.get("prefill") + decode_coverage = rules.get("decode") + if input_coverage == "tokens": + input_events = [event for event in events if event["phase"] == "input"] + if len(input_events) != 1 or input_events[0]["token_start"] != 0 or ( + input_events[0]["token_count"] != prompt_tokens): + raise TraceError(f"{component} input coverage is incomplete") + for layer in expected_layers or [None]: + layer_events = [event for event in events if event["layer"] == layer] + if prefill_coverage == "tokens": + prefill = sorted( + (event for event in layer_events if event["phase"] == "prefill"), + key=lambda event: event["token_start"], + ) + frontier = 0 + for event in prefill: + if event["token_start"] != frontier or event["token_count"] <= 0: + raise TraceError(f"{component} prefill coverage has a gap or overlap at token {frontier}") + frontier += event["token_count"] + if frontier != prompt_tokens: + raise TraceError( + f"{component} prefill coverage ends at {frontier}, expected {prompt_tokens}") + elif prefill_coverage == "final": + prefill = [event for event in layer_events if event["phase"] == "prefill"] + if len(prefill) != 1 or prefill[0]["token_start"] != prompt_tokens - 1 or prefill[0]["token_count"] != 1: + raise TraceError(f"{component} final prefill coverage is invalid") + if decode_coverage == "steps": + decode = sorted( + (event for event in layer_events if event["phase"] == "decode"), + key=lambda event: event["step"], + ) + if [event["step"] for event in decode] != list(range(decode_steps)): + raise TraceError(f"{component} decode step coverage is incomplete") + for event in decode: + if event["token_start"] != prompt_tokens + event["step"] or event["token_count"] != 1: + raise TraceError(f"{component} decode token coordinates are invalid") + + def _validate_component_schema(self, event: dict[str, Any]) -> None: + component = event["component"] + phase = event["phase"] + layer = event["layer"] + dtype = event["dtype"] + shape = event["shape"] + token_count = event["token_count"] + config = self.manifest["config"].get("deepseek41") + if not isinstance(config, dict): + raise TraceError("DeepSeek V4.1 component schema configuration is missing") + + if component == "prompt.bytes": + if phase != "input" or layer is not None or dtype != "bytes" or len(shape) != 1: + raise TraceError("prompt.bytes schema is invalid") + if shape[0] != self.manifest["prompt"]["byte_count"]: + raise TraceError("prompt.bytes length does not match the manifest") + if event["sha256"] != self.manifest["prompt"]["sha256"]: + raise TraceError("prompt.bytes SHA-256 does not match the manifest") + return + if component == "prompt.tokens": + if phase != "input" or layer is not None or dtype != "i32" or shape != [token_count]: + raise TraceError("prompt.tokens schema is invalid") + return + if component in ("logits.prefill", "logits.decode"): + if phase not in ("prefill", "decode") or layer is not None or dtype != "f32": + raise TraceError(f"{component} schema is invalid") + if token_count != 1 or shape != [config.get("vocab_size")]: + raise TraceError(f"{component} must contain one complete vocabulary-sized logit vector") + return + if component == "decode.greedy_token": + if phase != "decode" or layer is not None or dtype != "i32" or token_count != 1 or shape != [1]: + raise TraceError("decode.greedy_token schema is invalid") + return + + if phase not in ("prefill", "decode") or layer is None: + raise TraceError(f"{component} phase or layer is invalid") + if len(shape) != 2 or shape[1] != token_count: + raise TraceError(f"{component} second dimension must equal token_count") + widths = { + "engram.row_ids": ("i32", config.get("engram_rows_per_token")), + "expert.ids": ("i32", config.get("experts_used")), + "expert.weights": ("f32", config.get("experts_used")), + "attn.source": ("i32", None), + "attn.candidate_blocks": ("i32", None), + "attn.candidates": ("i32", None), + } + if component not in widths: + raise TraceError(f"unsupported trace component: {component}") + expected_dtype, width = widths[component] + if dtype != expected_dtype: + raise TraceError(f"{component} dtype must be {expected_dtype}") + if width is not None and shape[0] != width: + raise TraceError(f"{component} shape must be [{width},token_count]") + if component == "attn.source" and layer in RAW_ATTENTION_LAYERS: + if shape[0] != RAW_ATTENTION_WIDTH: + raise TraceError(f"raw attn.source shape must be [{RAW_ATTENTION_WIDTH},token_count]") + values = list(struct.iter_unpack(" config.get("candidate_topk_blocks", 0): + raise TraceError("attn.candidate_blocks width exceeds candidate_topk_blocks") + if component == "attn.source" and layer not in RAW_ATTENTION_LAYERS and ( + shape[0] > config.get("index_top_k", 0)): + raise TraceError(f"{component} width exceeds index_top_k") + if component == "attn.candidates" and shape[0] > config.get("index_top_k", 0): + raise TraceError(f"{component} width exceeds index_top_k") + if component == "expert.ids": + values = struct.iter_unpack("= config.get("expert_count", 0) for value, in values): + raise TraceError("expert.ids contains an out-of-range original expert ID") + + +_TRACE_BUNDLE_TYPE = TraceBundle + + +def first_byte_difference(left: bytes, right: bytes) -> int | None: + for index, (a, b) in enumerate(zip(left, right)): + if a != b: + return index + if len(left) != len(right): + return min(len(left), len(right)) + return None + + +def compare_manifests(left: TraceBundle, right: TraceBundle) -> Mismatch | None: + checks = ( + ("model.sha256", "model_identity"), + ("model.architecture", "model_identity"), + ("prompt.sha256", "prompt_identity"), + ("prompt.byte_count", "prompt_identity"), + ("expected.prompt_tokens", "tokenizer"), + ("config.context", "configuration"), + ("config.decode_steps", "configuration"), + ("config.deepseek41", "configuration"), + ("comparison.logits", "comparison_policy"), + ) + for dotted, classification in checks: + left_value: Any = left.manifest + right_value: Any = right.manifest + for key in dotted.split("."): + left_value = left_value.get(key) if isinstance(left_value, dict) else None + right_value = right_value.get(key) if isinstance(right_value, dict) else None + if left_value != right_value: + return Mismatch( + classification, + "manifest", + "metadata", + -1, + -1, + None, + f"{dotted} differs: {left_value!r} != {right_value!r}", + ) + return None + + +def compare_bundles( + left: TraceBundle, + right: TraceBundle, + *, + runtime_roles: tuple[str, str] = ("ds4", "llama.cpp")) -> Mismatch | None: + if left.root.resolve() == right.root.resolve(): + return Mismatch( + "artifact_identity", + "manifest", + "metadata", + -1, + -1, + None, + "cannot compare a trace bundle with itself", + ) + if left.manifest["runtime"] != runtime_roles[0] or right.manifest["runtime"] != runtime_roles[1]: + return Mismatch( + "runtime_role", + "manifest", + "metadata", + -1, + -1, + None, + f"left trace must be {runtime_roles[0]} and right trace must be {runtime_roles[1]}", + ) + mismatch = compare_manifests(left, right) + if mismatch is not None: + return mismatch + + for bundle_name, bundle in (("left", left), ("right", right)): + components = {event["component"] for event in bundle.events} + missing = sorted(set(HARD_FAILURE_COMPONENTS) - components) + if missing: + return Mismatch( + "artifact_missing", + "manifest", + "metadata", + -1, + -1, + None, + f"{bundle_name} trace is missing required components: {', '.join(missing)}", + ) + + left_map = {event_key(event): event for event in left.events} + right_map = {event_key(event): event for event in right.events} + if len(left_map) != len(left.events) or len(right_map) != len(right.events): + return Mismatch( + "artifact_duplicate", + "events", + "metadata", + -1, + -1, + None, + "a trace contains duplicate phase/step/token/layer/component coordinates", + ) + mismatches = [] + all_keys = sorted(set(left_map) | set(right_map), key=event_order) + for key in all_keys: + left_event = left_map.get(key) + right_event = right_map.get(key) + template = left_event or right_event + assert template is not None + if left_event is None or right_event is None: + mismatches.append(Mismatch( + "artifact_missing", + template["component"], + template["phase"], + template["step"], + template["token_start"], + template["layer"], + "event is missing from " + ("left" if left_event is None else "right") + " trace", + token_index=template["token_start"], + )) + continue + shape_mismatch = False + for field in ("dtype", "shape", "token_count", "semantic_id_space"): + if left_event.get(field) != right_event.get(field): + mismatches.append(Mismatch( + "artifact_shape", + template["component"], + template["phase"], + template["step"], + template["token_start"], + template["layer"], + f"{field} differs: {left_event.get(field)!r} != {right_event.get(field)!r}", + token_index=template["token_start"], + )) + shape_mismatch = True + break + if shape_mismatch: + continue + if left_event["sha256"] == right_event["sha256"]: + continue + left_data = left.read_blob(left_event) + right_data = right.read_blob(right_event) + byte_offset = first_byte_difference(left_data, right_data) + assert byte_offset is not None + item_size = DTYPE_SIZES[left_event["dtype"]] + flat_element_index = byte_offset // item_size + token_index = None + component_element_index = None + token_count = left_event["token_count"] + elements = element_count(left_event["shape"]) + if token_count > 0 and elements % token_count == 0: + elements_per_token = elements // token_count + token_index = left_event["token_start"] + flat_element_index // elements_per_token + component_element_index = flat_element_index % elements_per_token + detail = f"first byte mismatch at {byte_offset}" + item_offset = byte_offset - byte_offset % item_size + if item_offset + item_size <= min(len(left_data), len(right_data)): + left_item = left_data[item_offset:item_offset + item_size] + right_item = right_data[item_offset:item_offset + item_size] + detail += f"; left=0x{left_item.hex()} right=0x{right_item.hex()}" + mismatches.append(Mismatch( + classify(template["component"]), + template["component"], + template["phase"], + template["step"], + template["token_start"], + template["layer"], + detail, + element_index=flat_element_index, + byte_offset=byte_offset, + token_index=token_index, + component_element_index=component_element_index, + )) + if not mismatches: + return None + phase_order = {"input": 0, "prefill": 1, "decode": 2, "metadata": -1} + component_order = {component: index for index, component in enumerate(HARD_FAILURE_COMPONENTS)} + return min(mismatches, key=lambda item: ( + phase_order.get(item.phase, 99), + item.token_index if item.token_index is not None else item.token_start, + item.step, + -1 if item.layer is None else item.layer, + component_order.get(item.component, 99), + item.component, + )) + + +def report( + left: TraceBundle, + right: TraceBundle, + *, + runtime_roles: tuple[str, str] = ("ds4", "llama.cpp"), + success_status: str = "TARGET PASS") -> dict[str, Any]: + mismatch = compare_bundles(left, right, runtime_roles=runtime_roles) + if mismatch is None: + return { + "status": success_status, + "trace_version": TRACE_VERSION, + "left_runtime": left.manifest.get("runtime"), + "right_runtime": right.manifest.get("runtime"), + "events_compared": len(left.events), + "left_seal_sha256": left.seal_sha256, + "right_seal_sha256": right.seal_sha256, + "left_signer_principal": left.signer_principal, + "right_signer_principal": right.signer_principal, + "first_divergence": None, + } + return { + "status": "FAIL", + "trace_version": TRACE_VERSION, + "left_runtime": left.manifest.get("runtime"), + "right_runtime": right.manifest.get("runtime"), + "events_compared": 0, + "left_seal_sha256": left.seal_sha256, + "right_seal_sha256": right.seal_sha256, + "left_signer_principal": left.signer_principal, + "right_signer_principal": right.signer_principal, + "first_divergence": mismatch.as_dict(), + } + + +def command_validate(args: argparse.Namespace) -> int: + if hasattr(args, "approval_policy"): + approval_policy = load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(args.bundle,), + ) + bundle = TraceBundle( + args.bundle, + verifier=TraceVerifier.production( + args.signer_principal, + expected_lane=args.lane, + expected_challenge=args.execution_challenge, + expected_run_id=args.run_id, + expected_candidate_exporter_policy_id=args.candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=args.ds4_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + ), + ) + else: + bundle = TraceBundle(args.bundle) + print(canonical_json({ + "status": "valid", + "runtime": bundle.manifest.get("runtime"), + "events": len(bundle.events), + })) + return 0 + + +def command_compare(args: argparse.Namespace) -> int: + left_principal = getattr(args, "left_signer_principal", None) + right_principal = getattr(args, "right_signer_principal", None) + challenge = getattr(args, "execution_challenge", None) + left_run_id = getattr(args, "left_run_id", None) + right_run_id = getattr(args, "right_run_id", None) + approval_policy = ( + load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(args.left, args.right), + ) + if hasattr(args, "approval_policy") else None + ) + seen_run_ids: set[str] = set() + if approval_policy is None: + left_bundle = TraceBundle( + args.left, + signer_principal=left_principal, + expected_lane=ORACLE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=getattr(args, "left_ds4_exporter_policy_id", None), + expected_prompt_builder_policy_id=getattr(args, "prompt_builder_policy_id", None), + seen_run_ids=seen_run_ids, + ) + right_bundle = TraceBundle( + args.right, + signer_principal=right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=getattr( + args, "right_candidate_exporter_policy_id", None), + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=getattr(args, "prompt_builder_policy_id", None), + seen_run_ids=seen_run_ids, + ) + else: + left_bundle = TraceBundle( + args.left, + verifier=TraceVerifier.production( + left_principal, + expected_lane=ORACLE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=None, + expected_ds4_exporter_policy_id=args.left_ds4_exporter_policy_id, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + right_bundle = TraceBundle( + args.right, + verifier=TraceVerifier.production( + right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=args.right_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + result = report( + left_bundle, + right_bundle, + ) + text = canonical_json(result) + "\n" + if args.report: + args.report.write_text(text, encoding="ascii") + sys.stdout.write(text) + return 0 if result["status"] == "TARGET PASS" else 1 + + +def local_report(left: TraceBundle, right: TraceBundle, mode: str) -> dict[str, Any]: + result = report( + left, + right, + runtime_roles=("llama.cpp", "llama.cpp"), + success_status="BRINGUP PASS", + ) + result["mode"] = mode + result["cross_runtime_status"] = "INCOMPLETE" + result["cross_runtime_requirement"] = ( + "Run the pinned ds4 exporter and trace_format.py compare before reporting TARGET PASS.") + if mode == "self-consistency": + if left.manifest.get("candidate") != right.manifest.get("candidate"): + result = { + **result, + "status": "FAIL", + "first_divergence": Mismatch( + "candidate_identity", + "manifest", + "metadata", + -1, + -1, + None, + "self-consistency traces use different candidate attestations", + ).as_dict(), + } + else: + base_revision = left.manifest.get("candidate", {}).get("revision") + integrated_base = right.manifest.get("candidate", {}).get("base_revision") + if base_revision != integrated_base: + result = { + **result, + "status": "FAIL", + "first_divergence": Mismatch( + "candidate_identity", + "manifest", + "metadata", + -1, + -1, + None, + f"base trace revision {base_revision!r} != integrated oracle {integrated_base!r}", + ).as_dict(), + } + return result + + +def command_compare_local(args: argparse.Namespace) -> int: + left_principal = getattr(args, "left_signer_principal", None) + right_principal = getattr(args, "right_signer_principal", None) + challenge = getattr(args, "execution_challenge", None) + left_run_id = getattr(args, "left_run_id", None) + right_run_id = getattr(args, "right_run_id", None) + approval_policy = ( + load_executable_approval_policy( + args.approval_policy, + args.approval_signature, + expected_principal=args.approval_principal, + forbidden_roots=(args.left, args.right), + ) + if hasattr(args, "approval_policy") else None + ) + seen_run_ids: set[str] = set() + if approval_policy is None: + left_bundle = TraceBundle( + args.left, + signer_principal=left_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=getattr( + args, "left_candidate_exporter_policy_id", None), + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=getattr( + args, "left_prompt_builder_policy_id", None), + seen_run_ids=seen_run_ids, + ) + right_bundle = TraceBundle( + args.right, + signer_principal=right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=getattr( + args, "right_candidate_exporter_policy_id", None), + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=getattr( + args, "right_prompt_builder_policy_id", None), + seen_run_ids=seen_run_ids, + ) + else: + left_bundle = TraceBundle( + args.left, + verifier=TraceVerifier.production( + left_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=left_run_id, + expected_candidate_exporter_policy_id=args.left_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.left_prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + right_bundle = TraceBundle( + args.right, + verifier=TraceVerifier.production( + right_principal, + expected_lane=CANDIDATE_LANE, + expected_challenge=challenge, + expected_run_id=right_run_id, + expected_candidate_exporter_policy_id=args.right_candidate_exporter_policy_id, + expected_ds4_exporter_policy_id=None, + expected_prompt_builder_policy_id=args.right_prompt_builder_policy_id, + approval_policy=approval_policy, + verification_unix=None, + seen_run_ids=seen_run_ids, + ), + ) + result = local_report( + left_bundle, + right_bundle, + args.mode, + ) + text = canonical_json(result) + "\n" + if args.report: + args.report.write_text(text, encoding="ascii") + sys.stdout.write(text) + return 0 if result["status"] == "BRINGUP PASS" else 1 + + +def add_executable_approval_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--approval-policy", type=Path, required=True) + parser.add_argument("--approval-signature", type=Path, required=True) + parser.add_argument("--approval-principal", required=True) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Validate and compare DeepSeek V4.1 correctness traces") + subparsers = parser.add_subparsers(dest="command", required=True) + validate_parser = subparsers.add_parser("validate") + validate_parser.add_argument("bundle", type=Path) + validate_parser.add_argument("--signer-principal", required=True) + validate_parser.add_argument("--lane", choices=(CANDIDATE_LANE, ORACLE_LANE), required=True) + validate_parser.add_argument("--execution-challenge", required=True) + validate_parser.add_argument("--run-id", required=True) + validate_parser.add_argument("--candidate-exporter-policy-id") + validate_parser.add_argument("--ds4-exporter-policy-id") + validate_parser.add_argument("--prompt-builder-policy-id", required=True) + add_executable_approval_arguments(validate_parser) + validate_parser.set_defaults(func=command_validate) + compare_parser = subparsers.add_parser("compare") + compare_parser.add_argument("left", type=Path) + compare_parser.add_argument("right", type=Path) + compare_parser.add_argument("--left-signer-principal", required=True) + compare_parser.add_argument("--right-signer-principal", required=True) + compare_parser.add_argument("--execution-challenge", required=True) + compare_parser.add_argument("--left-run-id", required=True) + compare_parser.add_argument("--right-run-id", required=True) + compare_parser.add_argument("--left-ds4-exporter-policy-id", required=True) + compare_parser.add_argument("--right-candidate-exporter-policy-id", required=True) + compare_parser.add_argument("--prompt-builder-policy-id", required=True) + add_executable_approval_arguments(compare_parser) + compare_parser.add_argument("--report", type=Path) + compare_parser.set_defaults(func=command_compare) + local_parser = subparsers.add_parser("compare-local") + local_parser.add_argument("mode", choices=("self-consistency", "base-regression")) + local_parser.add_argument("left", type=Path) + local_parser.add_argument("right", type=Path) + local_parser.add_argument("--left-signer-principal", required=True) + local_parser.add_argument("--right-signer-principal", required=True) + local_parser.add_argument("--execution-challenge", required=True) + local_parser.add_argument("--left-run-id", required=True) + local_parser.add_argument("--right-run-id", required=True) + local_parser.add_argument("--left-candidate-exporter-policy-id", required=True) + local_parser.add_argument("--right-candidate-exporter-policy-id", required=True) + local_parser.add_argument("--left-prompt-builder-policy-id", required=True) + local_parser.add_argument("--right-prompt-builder-policy-id", required=True) + add_executable_approval_arguments(local_parser) + local_parser.add_argument("--report", type=Path) + local_parser.set_defaults(func=command_compare_local) + return parser + + +def main() -> int: + args = build_parser().parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/verify-runtime-install.py b/tools/deepseek-v41-trace/verify-runtime-install.py new file mode 100644 index 000000000000..b305ab1b7e5d --- /dev/null +++ b/tools/deepseek-v41-trace/verify-runtime-install.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +def strict_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--receipt", required=True, type=Path) + parser.add_argument("--install-root", required=True, type=Path) + args = parser.parse_args() + + receipt_bytes = args.receipt.read_bytes() + try: + receipt = json.loads(receipt_bytes, object_pairs_hook=strict_object) + except (UnicodeError, json.JSONDecodeError, ValueError) as error: + parser.error(f"runtime receipt JSON is invalid: {error}") + canonical = (json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + if receipt_bytes != canonical: + parser.error("runtime receipt JSON is not canonical") + if set(receipt) != {"format", "version", "revision", "profile", "components"} or ( + receipt["format"] != "dsv41-runtime-receipt" or receipt["version"] != 1 or + re.fullmatch(r"[0-9a-f]{40}", receipt["revision"]) is None or + receipt["profile"] not in {"co-located", "sibling-lib"}): + parser.error("runtime receipt is invalid") + directory = args.install_root / ("bin" if receipt["profile"] == "co-located" else "lib") + components = receipt["components"] + if not isinstance(components, list) or not components: + parser.error("runtime receipt component list is invalid") + if components != sorted(components, key=lambda item: item.get("component", "") if isinstance(item, dict) else ""): + parser.error("runtime receipt component list is not canonical") + names = set() + filenames = set() + digests = set() + for component in components: + if not isinstance(component, dict) or set(component) != { + "component", "filename", "sha256", "revision"}: + parser.error("runtime receipt component is invalid") + filename = component["filename"] + name = component["component"] + digest = component["sha256"] + revision = component["revision"] + if not isinstance(name, str) or re.fullmatch(r"[a-z0-9-]+", name) is None or name in names: + parser.error("runtime receipt component name is invalid") + if not isinstance(filename, str) or Path(filename).name != filename or ( + re.fullmatch(r"[A-Za-z0-9._+-]+", filename) is None) or filename in filenames: + parser.error("runtime receipt filename is invalid") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None or digest in digests: + parser.error("runtime receipt component SHA-256 is invalid") + revision_bearing = name in {"llama-common", "ggml-base"} + if (revision_bearing and revision != receipt["revision"]) or ( + not revision_bearing and revision is not None): + parser.error("runtime receipt component revision is invalid") + names.add(name) + filenames.add(filename) + digests.add(digest) + installed = directory / filename + if installed.is_symlink() or not installed.is_file(): + parser.error(f"installed runtime component is missing or not regular: {filename}") + if sha256_file(installed) != digest: + parser.error(f"installed runtime component SHA-256 mismatch: {name}") + if not {"llama-common", "llama", "ggml", "ggml-base"}.issubset(names): + parser.error("runtime receipt is missing core components") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deepseek-v41-trace/verify_ds4_anchors.py b/tools/deepseek-v41-trace/verify_ds4_anchors.py new file mode 100644 index 000000000000..5b8e552ae335 --- /dev/null +++ b/tools/deepseek-v41-trace/verify_ds4_anchors.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from trace_format import DS4_REVISION, sha256_file + + +ANCHORS = { + "tests/test-vectors/README.md": + "0e59b2f2832bed8af0a91e6ff20962debf964cd2d1d141c086e52cfcc995a1c3", + "tests/test-vectors/flash-0731/manifest.json": + "ebf237a5660a6851fb8085e77f532901a9d758208b25d7ed5af0b7af4b28f91b", + "tests/test-vectors/flash-0731/official.vec": + "77ae699889bfaf1348768dcbe7ea2c72279ae86abb10470d3e1b08cd1fd82a83", + "tests/test-vectors/flash-0731/local-golden.vec": + "23d942ff3b9bb2a3f82927d11aa3ed1461e1f302071e788d0d95a5c165e47d3b", + "tests/test_engram.c": + "198a561d981f62518a9d28035480a7e220b99c156cde6d248b8baabd684cc74b", + "ds4_engram.c": + "2b6ca468510ebf45ee298a905525bc7234dacad9a384bf2011eba19ba2c0bdf7", + "ds4_engram.h": + "f84a264e0fe199d23a6f0c56fbbd19e222adc7009eed13c185af6403f68f7f5c", + "tests/test_deepseek41_metal.c": + "9197c2f9d65b380ce25be5334991e4bfaf40e82e6708e64f2b9329552412d28c", + "tests/test_deepseek41_graph.c": + "6dc786f831c93ae7f5aa56e7518f67657125f7c0fd35cead3c646eff3d3f9e09", + "tests/test_deepseek41_prefill.c": + "452774b9332d393822d84288eeb2d71ca1fb25f1ba30d20a49160d1787de734f", + "tests/test_deepseek41_manifest.py": + "2d7aa1fc93805d9c97839c5de9eccc856628f13e6913c3bbc3f0c99b0827aeae", + "tests/test_deepseek41_conversion.py": + "a40b83062a9b91338773addd77fe62650296f4de607057cb4fd1329287f347b5c", + "tests/test_deepseek41_gguf.c": + "8f41e049d5ec179c38a1a00ac61712db0306902f0ef74bcdb989d8993316ff35", + "gguf-tools/deepseek41_metadata.py": + "39300bbd504165b97de017edd72563377f50b8a5511ca7478252a86f31a0009b", + "ds4.c": + "1776dbfed177ea14f3ce6cac1d8d0b1c1b44dfff2c2663769a9a5634aeec34e7", +} + + +class AnchorError(RuntimeError): + pass + + +def git_output(checkout: Path, *args: str) -> str: + try: + return subprocess.check_output( + ["git", "-C", str(checkout), *args], + text=True, + stderr=subprocess.STDOUT, + ).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise AnchorError(f"git {' '.join(args)} failed: {error}") from error + + +def verify( + checkout: Path, + *, + anchors: dict[str, str] = ANCHORS, + expected_revision: str = DS4_REVISION) -> dict[str, object]: + checkout = checkout.expanduser().resolve() + revision = git_output(checkout, "rev-parse", "HEAD") + if revision != expected_revision: + raise AnchorError(f"ds4 revision mismatch: expected {expected_revision}, found {revision}") + if git_output(checkout, "diff", "--name-only") or git_output(checkout, "diff", "--cached", "--name-only"): + raise AnchorError("ds4 tracked source differs from the pinned revision") + files = [] + for relative, expected in anchors.items(): + path = checkout / relative + if not path.is_file(): + raise AnchorError(f"pinned ds4 anchor is missing: {relative}") + actual = sha256_file(path) + if actual != expected: + raise AnchorError( + f"pinned ds4 anchor SHA-256 mismatch for {relative}: expected {expected}, found {actual}") + files.append({"path": relative, "sha256": actual}) + return { + "status": "ANCHORS VERIFIED", + "revision": revision, + "files": files, + "limitations": [ + "official.vec contains selected-token and top-logprob slices, not complete logits", + "local-golden.vec is a tolerant top-64 drift anchor", + "the Metal source fixture is not executable evidence on Strix", + "cross-runtime TARGET PASS still requires the pinned ds4 trace exporter", + ], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify pinned ds4 bring-up evidence anchors") + parser.add_argument( + "--checkout", + type=Path, + default=Path("/home/papa/src/ds4-v41"), + ) + args = parser.parse_args() + try: + print(json.dumps(verify(args.checkout), sort_keys=True, separators=(",", ":"))) + return 0 + except AnchorError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())