trace : recover hosted correctness blockers

Copilot-Session: c3ea1bf8-f288-47b4-9e1b-3435f2917bc1

Assisted-by: GPT-5.6 Sol

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: 2 people

CMakeLists.txt

@@ -131,6 +131,7 @@ option(LLAMA_BUILD_COMMON "llama: build common utils library" ${LLAMA_STANDALONE
 # extra artifacts
 option(LLAMA_BUILD_TESTS     "llama: build tests"                                                                ${LLAMA_STANDALONE})
 option(LLAMA_BUILD_TOOLS     "llama: build tools"                                                                ${LLAMA_STANDALONE})
+option(LLAMA_DEEPSEEK_V41_NATIVE_CONTAINMENT_TESTS "llama: run native Linux DeepSeek V4.1 containment tests"      OFF)
 option(LLAMA_BUILD_EXAMPLES  "llama: build examples"                                                             ${LLAMA_STANDALONE})
 option(LLAMA_BUILD_SERVER    "llama: build server example"                                                       ${LLAMA_STANDALONE})
 option(LLAMA_BUILD_APP       "llama: build the unified binary"                                                   ${LLAMA_STANDALONE})

tests/CMakeLists.txt

@@ -203,6 +203,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
     llama_build_and_test(test-deepseek41-memory.cpp)
     llama_build_and_test(test-deepseek41-runtime.cpp)
     llama_build_and_test(test-engram.cpp)
+    llama_build_and_test(test-server-child-protocol.cpp)
+    target_include_directories(test-server-child-protocol PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
     llama_build(test-llama-archs.cpp)
     llama_test(
         test-llama-archs
@@ -225,9 +227,17 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
                 TEST test-deepseek41-trace
                 APPEND PROPERTY ENVIRONMENT
                     "DSV41_NATIVE_TRACE_BINARY=$<TARGET_FILE:llama-deepseek-v41-trace>"
-                    "DSV41_NATIVE_CONTAINMENT_HELPER=$<TARGET_FILE:llama-deepseek-v41-containment-helper>"
                     "DSV41_NATIVE_MANIFEST_BINARY=$<TARGET_FILE:test-deepseek41-trace-manifest>"
                     "DSV41_NATIVE_INJECT_LIBRARY=$<TARGET_FILE:test-deepseek41-trace-injected>")
+            if(LLAMA_DEEPSEEK_V41_NATIVE_CONTAINMENT_TESTS)
+                if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+                    message(FATAL_ERROR "Native DeepSeek V4.1 containment tests require Linux")
+                endif()
+                set_property(
+                    TEST test-deepseek41-trace
+                    APPEND PROPERTY ENVIRONMENT
+                        "DSV41_NATIVE_CONTAINMENT_HELPER=$<TARGET_FILE:llama-deepseek-v41-containment-helper>")
+            endif()
         endif()
     endif()
 

tests/test-deepseek41-trace.py

@@ -258,6 +258,15 @@ def finish_native_test_supervisor(supervisor, *, kill_if_running):
     "runtime_libraries": DS4_RUNTIME_LIBRARIES,
     "runtime_libraries_post": copy.deepcopy(DS4_RUNTIME_LIBRARIES),
 }
+DS4_CONTAINMENT_HELPER = {
+    "format": "dsv41-containment-helper",
+    "version": 2,
+    "revision": trace.DS4_REVISION,
+    "filename": "llama-deepseek-v41-containment-helper",
+    "sha256": "d" * 64,
+    "launcher_policy": "zero-supplementary-groups-v1",
+    "supplementary_groups": [],
+}
 DS4_INSTALL_TRUST = {
     "format": "dsv41-install-trust",
     "version": 1,
@@ -303,6 +312,10 @@ def finish_native_test_supervisor(supervisor, *, kill_if_running):
         }
         for index, (path, digest) in enumerate(
             sorted((
+                (
+                    "/Users/oracle/ds4-install/bin/llama-deepseek-v41-containment-helper",
+                    DS4_CONTAINMENT_HELPER["sha256"],
+                ),
                 ("/Users/oracle/ds4-install/bin/ds4-trace", FIXTURE_DS4_EXPORTER_SHA256),
                 ("/Users/oracle/ds4-install/lib/libds4-runtime.dylib", "a" * 64),
                 ("/Users/oracle/ds4-install/lib/libds4-metal.dylib", "b" * 64),
@@ -319,6 +332,7 @@ def finish_native_test_supervisor(supervisor, *, kill_if_running):
     "install_owner_uid": 0,
     "executable_path": "/Users/oracle/ds4-install/bin/ds4-trace",
     "executable_sha256": FIXTURE_DS4_EXPORTER_SHA256,
+    "containment_helper": DS4_CONTAINMENT_HELPER,
     "runtime_profile": DS4_RUNTIME_PROFILE,
     "runtime_receipt": DS4_RUNTIME_RECEIPT,
 }
@@ -2056,6 +2070,14 @@ def test_ds4_approval_rejects_runtime_identity_mismatches(self) -> None:
                     lambda value: value["runtime_receipt"].update({"profile": "co-located"}),
                     "runtime receipt identity",
                 ),
+                (
+                    lambda value: value.pop("containment_helper"),
+                    "missing containment_helper",
+                ),
+                (
+                    lambda value: value["containment_helper"].update({"revision": "a" * 40}),
+                    "containment helper receipt",
+                ),
                 (
                     lambda value: [
                         component.update({"revision": None})
@@ -2203,11 +2225,16 @@ def test_approved_executable_postchecks_before_strict_decode(self) -> None:
             executable.parent.mkdir(parents=True)
             executable.write_bytes(b"approved")
             executable.chmod(0o555)
+            revision = "a" * 40
             policy = {
+                "revision": revision,
                 "install_root": str(install),
                 "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0,
+                "executable_path": str(executable),
+                "containment_helper": fixture_containment_helper(revision),
                 "runtime_receipt": {"components": []},
             }
+            materialize_policy_runtime(policy)
             events = []
             completed = subprocess.CompletedProcess([str(executable)], 0, b"\xff", b"")
             contained = trace._ContainedRun(completed, None, [], None, True, True)
@@ -4176,11 +4203,16 @@ def test_containment_handle_close_failure_forces_quiescence_false(self) -> None:
             executable.parent.mkdir(parents=True)
             executable.write_bytes(b"approved")
             executable.chmod(0o555)
+            revision = "a" * 40
             policy = {
+                "revision": revision,
                 "install_root": str(install),
                 "install_owner_uid": os.geteuid() if hasattr(os, "geteuid") else 0,
+                "executable_path": str(executable),
+                "containment_helper": fixture_containment_helper(revision),
                 "runtime_receipt": {"components": []},
             }
+            materialize_policy_runtime(policy)
             result = subprocess.CompletedProcess([str(executable)], 0, b"", b"")
             contained = trace._ContainedRun(result, None, [], mock.Mock(), True, True)
             close_failure = trace._IntegrityFailure(
@@ -4336,6 +4368,43 @@ def test_ds4_policy_and_receipt_mutation_change_approval_identity(self) -> None:
                 seen_run_ids=None,
             )
 
+    def test_ds4_install_trust_producer_binds_containment_helper(self) -> None:
+        with tempfile.TemporaryDirectory() as temp:
+            policy, exporter = materialize_ds4_exporter_policy(Path(temp).resolve())
+            with isolated_test_install_trust():
+                exporter_identity = run_ds4.approved_executable_identity(
+                    exporter,
+                    install_root=policy["install_root"],
+                    expected_owner_uid=policy["install_owner_uid"],
+                    expected_path=policy["executable_path"],
+                    expected_sha256=policy["executable_sha256"],
+                    label="ds4 exporter",
+                )
+                runtime_identities = run_ds4.approved_runtime_file_identities(
+                    policy, label="ds4 exporter")
+                trust = run_ds4.exporter_install_trust_evidence(
+                    exporter_identity, runtime_identities, policy)
+                self.assertEqual(
+                    trace.validate_install_trust_evidence(trust, policy),
+                    trust,
+                )
+                self.assertIn(
+                    str(Path(policy["install_root"]) / "bin" / policy["containment_helper"]["filename"]),
+                    {item["path"] for item in trust["files"]},
+                )
+
+                omitted = trace.install_trust_evidence(
+                    exporter_identity, runtime_identities)
+                with self.assertRaisesRegex(
+                        trace.TraceError, "files differ from external approval"):
+                    trace.validate_install_trust_evidence(omitted, policy)
+
+                wrong_helper = copy.deepcopy(policy)
+                wrong_helper["containment_helper"]["sha256"] = "e" * 64
+                with self.assertRaisesRegex(run_ds4.TraceError, "SHA-256 differs"):
+                    run_ds4.exporter_install_trust_evidence(
+                        exporter_identity, runtime_identities, wrong_helper)
+
     def test_install_trust_evidence_rejects_mutability_claims(self) -> None:
         policy = fixture_prompt_builder_policy(b"prompt")
         mutations = {

tests/test-server-child-protocol.cpp

@@ -0,0 +1,22 @@
+#include "server-child-protocol.h"
+
+#include <cassert>
+#include <string>
+#include <string_view>
+
+int main() {
+    constexpr std::string_view state =
+        "cmd_child_to_router:state:{\"state\":\"downloading\",\"payload\":{\"result\":\"download_finished\"}}\n";
+
+    const std::string reset_state = std::string("\x1b[0m") + std::string(state);
+    const std::string repeated_reset_state = std::string("\x1b[0m\x1b[0m") + std::string(state);
+    const std::string colored_state = std::string("\x1b[31m") + std::string(state);
+
+    assert(server_child_state_line(state) == state);
+    assert(server_child_state_line(reset_state) == state);
+    assert(server_child_state_line(repeated_reset_state) == state);
+    assert(server_child_state_line(colored_state).empty());
+    assert(server_child_state_line("ordinary child log\n").empty());
+    assert(server_child_state_line("\x1b[0cmd_child_to_router:state:{}\n").empty());
+    return 0;
+}

tools/deepseek-v41-trace/README.md

@@ -118,6 +118,8 @@ build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o SET_ROWS
 build-dsv41-trace-rocm/bin/test-backend-ops -b ROCm0 -o CPY
 ```
 
+Native Linux containment tests are not enabled by default because generic hosted runners do not provide the required mount policy. A dedicated single-purpose runner must set `-DLLAMA_DEEPSEEK_V41_NATIVE_CONTAINMENT_TESTS=ON` and use exactly one administrator-provisioned host configuration: `kernel.apparmor_restrict_unprivileged_userns=0`, or a narrowly scoped AppArmor allow policy for the unchanged receipt-bound helper user namespace, `MS_PRIVATE`, private procfs, and verification sequence. Exact AppArmor policy syntax is host-specific and is not supplied here. The test and production launcher still fail closed when the required namespace operations are unavailable.
+
 Do not change host ROCm packages for this run. Vulkan can provide secondary coverage, but it cannot replace the required ROCm low-level and oracle evidence. The llama runner selects `ROCm0` explicitly, invokes the exact exporter for a pre-allocation device attestation, and rejects the run unless the backend PCI identity maps to exactly one KFD node reporting `gfx1151`. The native exporter repeats the query before model allocation and verifies that the loaded model still uses the same device.
 
 Static repository builds skip this shared-library trace component instead of failing configuration.

tools/deepseek-v41-trace/run_ds4.py

@@ -38,6 +38,7 @@
     TraceError,
     TraceVerifier,
     approval_binding,
+    approved_containment_helper_identity,
     approved_executable_identity,
     approved_runtime_file_identities,
     bind_execution_authorization,
@@ -84,6 +85,16 @@ def __init__(
         self.secondary_errors = tuple(secondary_errors)
 
 
+def exporter_install_trust_evidence(
+        exporter_identity: ExecutableFileReceipt,
+        runtime_identities: list[ExecutableFileReceipt],
+        exporter_policy: dict[str, Any]) -> dict[str, Any]:
+    helper_identity = approved_containment_helper_identity(
+        exporter_policy, label="ds4 exporter")
+    return install_trust_evidence(
+        exporter_identity, runtime_identities, (helper_identity,))
+
+
 def git_output(checkout: Path, *args: str) -> str:
     try:
         return subprocess.check_output(
@@ -593,8 +604,8 @@ def main() -> int:
         )
         runtime_identities = approved_runtime_file_identities(
             exporter_policy, label="ds4 exporter")
-        exporter_install_trust = install_trust_evidence(
-            exporter_identity, runtime_identities)
+        exporter_install_trust = exporter_install_trust_evidence(
+            exporter_identity, runtime_identities, exporter_policy)
         exporter_install_trust_sha256 = install_trust_sha256(exporter_install_trust)
         pre_runtime_build = query_runtime_build_attestation(
             exporter,

tools/deepseek-v41-trace/trace_format.py

@@ -2668,6 +2668,7 @@ def ds4_exporter_approval(
             "install_owner_uid",
             "executable_path",
             "executable_sha256",
+            "containment_helper",
             "runtime_profile",
             "runtime_receipt",
         },
@@ -2686,6 +2687,12 @@ def ds4_exporter_approval(
     executable = PurePosixPath(executable_path)
     if executable.parent != PurePosixPath(install_root) / "bin":
         raise TraceError("ds4 exporter approval executable path is outside its install policy")
+    _validate_containment_helper_policy(
+        policy["containment_helper"],
+        install_root=install_root,
+        revision=policy["revision"],
+        label="ds4 exporter approval",
+    )
     profile = policy["runtime_profile"]
     if not isinstance(profile, dict):
         raise TraceError("ds4 exporter approval runtime profile is invalid")

tools/server/server-child-protocol.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string_view>
+
+inline constexpr std::string_view SERVER_CHILD_STATE_PREFIX = "cmd_child_to_router:state:";
+
+inline std::string_view server_child_state_line(std::string_view line) {
+    constexpr std::string_view ansi_reset = "\x1b[0m";
+
+    // Colored logs write the reset after the newline, before the next raw protocol line.
+    while (line.compare(0, ansi_reset.size(), ansi_reset) == 0) {
+        line.remove_prefix(ansi_reset.size());
+    }
+    if (line.compare(0, SERVER_CHILD_STATE_PREFIX.size(), SERVER_CHILD_STATE_PREFIX) != 0) {
+        return {};
+    }
+    return line;
+}

tools/server/server-models.cpp

@@ -1,4 +1,5 @@
 #include "server-common.h"
+#include "server-child-protocol.h"
 #include "http.h"
 #include "server-models.h"
 #include "server-context.h"
@@ -42,7 +43,6 @@ extern char **environ;
 #define DEFAULT_STOP_TIMEOUT 10 // seconds
 
 #define CMD_ROUTER_TO_CHILD_EXIT  "cmd_router_to_child:exit"
-#define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string
 
 // address for child process, this is needed because router may run on 0.0.0.0
 // ref: https://github.com/ggml-org/llama.cpp/issues/17862
@@ -1053,9 +1053,10 @@ void server_models::load(const std::string & name, const load_options & opts) {
             if (stdout_file) {
                 while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) {
                     std::string str(buffer);
-                    if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) {
-                        LOG_DBG("[%5d] %s", port, buffer); // prevent spamming the log
-                        this->handle_child_state(name, str);
+                    std::string state_line(server_child_state_line(str));
+                    if (!state_line.empty()) {
+                        LOG_DBG("[%5d] %s", port, state_line.c_str()); // prevent spamming the log
+                        this->handle_child_state(name, state_line);
                     } else {
                         // forward log
                         LOG("[%5d] %s", port, buffer);
@@ -1110,7 +1111,10 @@ void server_models::load(const std::string & name, const load_options & opts) {
         }
 
         child_proc->stopped.store(true, std::memory_order_release);
-        {
+        if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) {
+            // Download children never enter stopping_models. Avoid the model lock after DOWNLOADED so remove() can join.
+            cv_stop.notify_all();
+        } else {
             std::lock_guard<std::mutex> lk(this->mutex);
             stopping_models.erase(name);
             cv_stop.notify_all();
@@ -1525,7 +1529,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
     json payload;
 
     try {
-        json data = json::parse(raw_input.substr(strlen(CMD_CHILD_TO_ROUTER_STATE)));
+        json data = json::parse(raw_input.substr(SERVER_CHILD_STATE_PREFIX.size()));
         state = server_state_from_str(json_value(data, "state", std::string()));
         payload = json_value(data, "payload", json{});
     } catch (const std::exception & e) {
@@ -1716,7 +1720,7 @@ void server_child::notify_to_router(const std::string & state, const json & payl
     std::lock_guard<std::mutex> lk(mtx_stdout);
     common_log_pause(common_log_main());
     fflush(stdout);
-    fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());
+    fprintf(stdout, "%s%s\n", SERVER_CHILD_STATE_PREFIX.data(), safe_json_to_str(data).c_str());
     fflush(stdout);
     common_log_resume(common_log_main());
 }

tools/server/tests/unit/test_router.py

@@ -504,6 +504,11 @@ def test_router_remote_preset():
 MODEL_DOWNLOAD_TIMEOUT = 30
 
 
+@pytest.fixture
+def private_router_download_cache(monkeypatch, tmp_path):
+    monkeypatch.setenv("LLAMA_CACHE", str(tmp_path / "llama-cache"))
+
+
 def _listen_sse(
     server: ServerProcess, collected: list, stop: threading.Event, ready: threading.Event | None = None
 ):
@@ -539,7 +544,7 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i
     return False
 
 
-def test_router_download_model():
+def test_router_download_model(private_router_download_cache):
     """Case 1: download a model, verify SSE events and GET /models."""
     global server
     server.start()
@@ -582,7 +587,7 @@ def test_router_download_model():
     assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download"
 
 
-def test_router_delete_model():
+def test_router_delete_model(private_router_download_cache):
     """Case 2: delete the downloaded model, verify it disappears from GET /models."""
     global server
     server.start()