diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index b0088d5a9..65c88a744 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -70,7 +70,6 @@ sources = files( 'value/context.cc', 'value/print.cc', 'value/value.cc', - 'worldtree-pool.cc', ) include_dirs = [ include_directories('.') ] diff --git a/src/libexpr-tests/tectonix.cc b/src/libexpr-tests/tectonix.cc index b41b10ebc..3410906d9 100644 --- a/src/libexpr-tests/tectonix.cc +++ b/src/libexpr-tests/tectonix.cc @@ -145,7 +145,10 @@ class TectonixTest : public ::testing::Test std::unique_ptr state; TectonixEvalContext( - const std::filesystem::path & repoPath, const std::string & commitSha, bool withCheckout = false) + const std::filesystem::path & repoPath, + const std::string & commitSha, + bool withCheckout = false, + const std::filesystem::path & worldtreeMount = {}) : store(openStore("dummy://")) { evalSettings.nixPath = {}; @@ -154,6 +157,11 @@ class TectonixTest : public ::testing::Test if (withCheckout) { evalSettings.tectonixCheckoutPath = repoPath.string(); } + if (!worldtreeMount.empty()) { + // Deliberately nonexistent: historical mode must never connect to it. + evalSettings.tectonixWorldtreeSocket = (repoPath / "unused-worldtree.sock").string(); + evalSettings.tectonixWorldtreeMount = worldtreeMount.string(); + } state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings, nullptr); } @@ -172,6 +180,15 @@ class TectonixTest : public ::testing::Test { return std::make_unique(repoPath, commitSha, withCheckout); } + + std::unique_ptr createHistoricalWorldtreeContext(std::string_view manifest) + { + auto mount = repoPath / "worldtree"; + auto manifestDir = mount / "tecnix" / commitSha / "W-000000"; + std::filesystem::create_directories(manifestDir); + std::ofstream(manifestDir / "manifest.json") << manifest; + return std::make_unique(repoPath, commitSha, false, mount); + } }; // ============================================================================ @@ -211,6 +228,69 @@ TEST_F(TectonixTest, manifest_returns_path_to_metadata_mapping) ASSERT_THAT(*coreId->value, IsStringEq("W-000003")); } +TEST_F(TectonixTest, historical_worldtree_manifest_and_dirty_set_are_filesystem_only) +{ + static constexpr std::string_view historicalManifest = R"({ + "//historical/only": { "id": "W-123456" } + })"; + auto ctx = createHistoricalWorldtreeContext(historicalManifest); + + ASSERT_EQ(ctx->state->getManifestContent(), historicalManifest); + auto & manifest = ctx->state->getManifestJson(); + ASSERT_EQ(manifest.size(), 1u); + ASSERT_EQ(manifest.at("//historical/only").at("id"), "W-123456"); + + auto & dirty = ctx->state->getTectonixDirtyZones(); + ASSERT_EQ(dirty.size(), 1u); + ASSERT_FALSE(dirty.at("//historical/only").dirty); +} + +TEST_F(TectonixTest, historical_worldtree_explains_malformed_manifest) +{ + auto ctx = createHistoricalWorldtreeContext("{"); + + try { + ctx->state->getManifestJson(); + FAIL() << "expected malformed manifest to fail"; + } catch (const Error & e) { + ASSERT_THAT(e.what(), testing::HasSubstr("historical World manifest '.meta/manifest.json'")); + ASSERT_THAT(e.what(), testing::HasSubstr(commitSha)); + ASSERT_THAT(e.what(), testing::HasSubstr("missing or malformed")); + } +} + +TEST_F(TectonixTest, historical_worldtree_explains_missing_manifest) +{ + auto mount = repoPath / "worldtree-without-manifest"; + auto ctx = std::make_unique(repoPath, commitSha, false, mount); + + try { + ctx->state->getManifestContent(); + FAIL() << "expected missing manifest to fail"; + } catch (const Error & e) { + ASSERT_THAT(e.what(), testing::HasSubstr("historical World manifest '.meta/manifest.json'")); + ASSERT_THAT(e.what(), testing::HasSubstr(commitSha)); + ASSERT_THAT(e.what(), testing::HasSubstr("missing or malformed")); + } +} + +TEST_F(TectonixTest, historical_worldtree_rejects_noncanonical_revision) +{ + auto mount = repoPath / "worldtree"; + auto ctx = std::make_unique(repoPath, "../escape", false, mount); + + ASSERT_THROW(ctx->state->getManifestContent(), Error); +} + +TEST_F(TectonixTest, historical_worldtree_rejects_noncanonical_zone_id) +{ + auto ctx = createHistoricalWorldtreeContext(R"({ + "//historical/only": { "id": "../escape" } + })"); + + ASSERT_THROW(ctx->state->getZoneStorePath("//historical/only"), Error); +} + // ============================================================================ // Phase 4: Builtin Tests - __unsafeTectonixInternalManifestInverted // ============================================================================ diff --git a/src/libexpr-tests/worldtree-pool.cc b/src/libexpr-tests/worldtree-pool.cc deleted file mode 100644 index a0b4fa2c8..000000000 --- a/src/libexpr-tests/worldtree-pool.cc +++ /dev/null @@ -1,467 +0,0 @@ -#include - -#include "nix/expr/eval.hh" -#include "nix/expr/eval-settings.hh" -#include "nix/fetchers/fetch-settings.hh" -#include "nix/store/globals.hh" -#include "nix/store/store-open.hh" -#include "nix/util/file-system.hh" -#include "nix/util/hash.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -// ============================================================================ -// Worldtree connection-pool tests (the accessor side) -// -// worldtreed's own suite covers the daemon-side session invariants; these cover the -// *client* pool in `EvalState::acquireWorldtreeZoneSession` -- the logic most likely to -// silently regress to one-connection-per-zone: lazy growth bounded by -// `tectonix-worldtree-max-connections`, round-robin reuse past the cap, and release of -// each per-zone session when its accessor handle drops (a `scoped.close_ro` on the -// hosting connection). We drive the *real* Client + pool + session RAII against a minimal -// in-process fake worldtreed speaking the real control-socket wire, so nothing about the -// production path is stubbed. -// ============================================================================ - -namespace nix { - -namespace { - -// ---- worldtree wire helpers (mirror libutil/worldtree-client.cc, server direction) ---- - -constexpr uint32_t WT_F_REQUEST_ID = 1; -constexpr uint32_t WT_F_KIND = 2; -constexpr uint32_t WT_F_METHOD = 3; -constexpr uint32_t WT_F_PAYLOAD = 4; -constexpr uint64_t WT_KIND_RESPONSE = 2; - -void wtPutVarint(std::string & out, uint64_t v) -{ - while (v >= 0x80) { - out.push_back(static_cast((v & 0x7f) | 0x80)); - v >>= 7; - } - out.push_back(static_cast(v)); -} - -/// Singular varint field, omitted when zero (proto3 default) -- matches the client encoder. -void wtPutVarintField(std::string & out, uint32_t field, uint64_t v) -{ - if (v == 0) - return; - wtPutVarint(out, (static_cast(field) << 3) | 0); - wtPutVarint(out, v); -} - -/// Singular length-delimited field, omitted when empty (proto3 default). -void wtPutLenField(std::string & out, uint32_t field, std::string_view bytes) -{ - if (bytes.empty()) - return; - wtPutVarint(out, (static_cast(field) << 3) | 2); - wtPutVarint(out, bytes.size()); - out.append(bytes); -} - -uint64_t wtReadVarintBuf(const std::string & buf, size_t & p) -{ - uint64_t v = 0; - int shift = 0; - while (p < buf.size()) { - uint8_t b = static_cast(buf[p++]); - v |= static_cast(b & 0x7f) << shift; - if (!(b & 0x80)) - break; - shift += 7; - } - return v; -} - -bool wtReadExact(int fd, char * buf, size_t n) -{ - size_t off = 0; - while (off < n) { - ssize_t r = ::read(fd, buf + off, n - off); - if (r < 0) { - if (errno == EINTR) - continue; - return false; - } - if (r == 0) - return false; // peer closed - off += static_cast(r); - } - return true; -} - -void wtWriteAll(int fd, const std::string & bytes) -{ - size_t off = 0; - while (off < bytes.size()) { - ssize_t w = ::write(fd, bytes.data() + off, bytes.size() - off); - if (w < 0) { - if (errno == EINTR) - continue; - return; - } - off += static_cast(w); - } -} - -/// A minimal in-process fake worldtree daemon. It answers only the two session verbs the -/// pool exercises -- `scoped.open_ro` (with a fresh, monotonic session id) and -/// `scoped.close_ro` (recording the released id) -- and records how many connections were -/// accepted and which sessions opened on each, so a test can assert the accessor pool's -/// bounded, round-robin, close-on-drop behaviour. Not a general daemon; every other verb -/// gets a benign empty reply (never reached by these tests). -class FakeWorldtreeDaemon -{ -public: - explicit FakeWorldtreeDaemon(std::string path) - : path_(std::move(path)) - { - listenFd_ = ::socket(AF_UNIX, SOCK_STREAM, 0); - assert(listenFd_ >= 0); - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - assert(path_.size() < sizeof(addr.sun_path)); - std::memcpy(addr.sun_path, path_.c_str(), path_.size()); - ::unlink(path_.c_str()); - assert(::bind(listenFd_, reinterpret_cast(&addr), sizeof(addr)) == 0); - assert(::listen(listenFd_, 128) == 0); - acceptThread_ = std::thread([this] { acceptLoop(); }); - } - - ~FakeWorldtreeDaemon() - { - stop_ = true; - // Wake a blocked accept() with a throwaway self-connect, then join. - int w = ::socket(AF_UNIX, SOCK_STREAM, 0); - if (w >= 0) { - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - std::memcpy(addr.sun_path, path_.c_str(), path_.size()); - (void) ::connect(w, reinterpret_cast(&addr), sizeof(addr)); - ::close(w); - } - if (acceptThread_.joinable()) - acceptThread_.join(); - // Unblock any worker still reading (if a client fd has not closed yet), join, close. - for (int fd : connFds_) - ::shutdown(fd, SHUT_RDWR); - for (auto & t : workers_) - if (t.joinable()) - t.join(); - for (int fd : connFds_) - ::close(fd); - if (listenFd_ >= 0) - ::close(listenFd_); - ::unlink(path_.c_str()); - } - - const std::string & path() const - { - return path_; - } - - size_t connectionCount() - { - std::lock_guard l(mtx_); - return sessionsPerConn_.size(); - } - - /// Per-accepted-connection, the session ids opened on it, in acceptance order. - std::vector> sessionsPerConn() - { - std::lock_guard l(mtx_); - return sessionsPerConn_; - } - - std::set closedSessions() - { - std::lock_guard l(mtx_); - return closed_; - } - -private: - void acceptLoop() - { - for (;;) { - int fd = ::accept(listenFd_, nullptr, nullptr); - if (fd < 0) - return; - if (stop_) { - ::close(fd); - return; - } - size_t idx; - { - std::lock_guard l(mtx_); - idx = sessionsPerConn_.size(); - sessionsPerConn_.emplace_back(); - connFds_.push_back(fd); - } - workers_.emplace_back([this, fd, idx] { serve(fd, idx); }); - } - } - - void serve(int fd, size_t idx) - { - for (;;) { - uint64_t reqId = 0; - std::string method, payload; - if (!readFrame(fd, reqId, method, payload)) - return; // client closed the connection - if (method == "scoped.open_ro") { - uint64_t session; - { - std::lock_guard l(mtx_); - session = nextSession_++; - sessionsPerConn_[idx].push_back(session); - } - std::string pl; - wtPutVarintField(pl, 1, session); // OpenRoResp { ws = 1 } - sendResponse(fd, reqId, pl); - } else if (method == "scoped.close_ro") { - uint64_t ws = firstVarintField(payload); // CloseRoReq { ws = 1 } - { - std::lock_guard l(mtx_); - closed_.insert(ws); - } - sendResponse(fd, reqId, {}); // CloseRoResp is empty - } else { - sendResponse(fd, reqId, {}); - } - } - } - - /// Value of field 1 (varint) in a message, or 0 if absent. - static uint64_t firstVarintField(const std::string & body) - { - size_t p = 0; - while (p < body.size()) { - uint64_t tag = wtReadVarintBuf(body, p); - if ((tag >> 3) == 1 && (tag & 0x7) == 0) - return wtReadVarintBuf(body, p); - break; - } - return 0; - } - - bool readFrame(int fd, uint64_t & reqId, std::string & method, std::string & payload) - { - // Length-delimited: a LEB128 varint byte count, then the frame body. - uint64_t len = 0; - int shift = 0; - for (;;) { - char c; - if (!wtReadExact(fd, &c, 1)) - return false; - uint8_t b = static_cast(c); - len |= static_cast(b & 0x7f) << shift; - if (!(b & 0x80)) - break; - shift += 7; - } - std::string body(len, '\0'); - if (len && !wtReadExact(fd, body.data(), len)) - return false; - reqId = 0; - method.clear(); - payload.clear(); - size_t p = 0; - while (p < body.size()) { - uint64_t tag = wtReadVarintBuf(body, p); - uint32_t field = static_cast(tag >> 3); - uint32_t wire = static_cast(tag & 0x7); - if (wire == 0) { - uint64_t v = wtReadVarintBuf(body, p); - if (field == WT_F_REQUEST_ID) - reqId = v; - } else if (wire == 2) { - uint64_t n = wtReadVarintBuf(body, p); - std::string s = body.substr(p, n); - p += n; - if (field == WT_F_METHOD) - method = std::move(s); - else if (field == WT_F_PAYLOAD) - payload = std::move(s); - } else { - break; // no other wire types on the request path - } - } - return true; - } - - void sendResponse(int fd, uint64_t reqId, std::string_view payload) - { - std::string frame; - wtPutVarintField(frame, WT_F_REQUEST_ID, reqId); - wtPutVarintField(frame, WT_F_KIND, WT_KIND_RESPONSE); - wtPutLenField(frame, WT_F_PAYLOAD, payload); - std::string wire; - wtPutVarint(wire, frame.size()); - wire += frame; - wtWriteAll(fd, wire); - } - - std::string path_; - int listenFd_ = -1; - std::thread acceptThread_; - std::atomic stop_{false}; - std::vector workers_; - - std::mutex mtx_; - std::vector connFds_; - std::vector> sessionsPerConn_; - std::set closed_; - uint64_t nextSession_ = 1; -}; - -} // namespace - -class WorldtreePoolTest : public ::testing::Test -{ -protected: - static void SetUpTestSuite() - { - initLibStore(false); // idempotent (guarded internally) - initGC(); // idempotent (guarded internally) - } - - void SetUp() override - { - tmpDir = createTempDir(); - delTmpDir = std::make_unique(tmpDir, true); - } - - void TearDown() override - { - delTmpDir.reset(); - } - - /// Keeps an EvalState and the settings/store it borrows alive together. - struct Ctx - { - bool readOnly = true; - fetchers::Settings fetchSettings{}; - EvalSettings evalSettings{readOnly}; - ref store = openStore("dummy://"); - std::unique_ptr state; - - EvalState & wire(const std::string & socketPath, uint64_t cap) - { - evalSettings.nixPath = {}; - evalSettings.tectonixWorldtreeSocket = socketPath; - evalSettings.tectonixWorldtreeWorkspace = 42; - evalSettings.tectonixWorldtreeMaxConnections = cap; - state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings, nullptr); - return *state; - } - }; - - /// Friend-access shim: only `WorldtreePoolTest` may reach the private pool seam, so the - /// TEST_F bodies (which run in a derived class) call this rather than the private method. - std::shared_ptr acquire(EvalState & s, const Hash & sha, const std::string & zone) - { - return s.acquireWorldtreeZoneSession(sha, zone); - } - - std::filesystem::path tmpDir; - std::unique_ptr delTmpDir; -}; - -// M > cap zones must fan out over at most `cap` physical connections (lazy growth, then -// round-robin reuse), never one connection per zone. -TEST_F(WorldtreePoolTest, bounds_connections_and_round_robins) -{ - const uint64_t cap = 4; - const int zones = 20; - - // Order matters: `state` (in ctx) is torn down before `fake`, so client fds close and - // the fake's workers see EOF before the daemon joins them. - FakeWorldtreeDaemon fake((tmpDir / "w.sock").string()); - Ctx ctx; - EvalState & state = ctx.wire(fake.path(), cap); - - Hash sha(HashAlgorithm::SHA1); // zero commit oid; the fake ignores it - - std::vector> handles; - - // Lazy growth: the pool must not pre-allocate. After k < cap acquisitions there are - // exactly k connections. - handles.push_back(acquire(state, sha, "//areas/z0")); - EXPECT_EQ(fake.connectionCount(), 1u); - handles.push_back(acquire(state, sha, "//areas/z1")); - EXPECT_EQ(fake.connectionCount(), 2u); - - for (int i = 2; i < zones; ++i) - handles.push_back(acquire(state, sha, "//areas/z" + std::to_string(i))); - - // Bounded: exactly `cap` connections for M > cap zones (the core regression guard). - EXPECT_EQ(fake.connectionCount(), cap); - - auto perConn = fake.sessionsPerConn(); - ASSERT_EQ(perConn.size(), cap); - - // Every zone got its own session, and the ids cover 1..M with none lost or duplicated. - std::set all; - size_t total = 0; - for (const auto & sessions : perConn) { - total += sessions.size(); - all.insert(sessions.begin(), sessions.end()); - } - EXPECT_EQ(total, static_cast(zones)); - EXPECT_EQ(all.size(), static_cast(zones)); - - // Round-robin: sessions are handed to connections cyclically, so connection i hosts - // exactly the sessions whose (id-1) % cap == i, and each connection is used evenly. - for (size_t i = 0; i < perConn.size(); ++i) { - for (uint64_t session : perConn[i]) - EXPECT_EQ((session - 1) % cap, i) << "session " << session << " on connection " << i; - EXPECT_GE(perConn[i].size(), static_cast(zones) / cap); - } -} - -// Dropping the accessor handles must release each per-zone session (a `scoped.close_ro` on -// the hosting connection), not leak them on the pooled connections. -TEST_F(WorldtreePoolTest, closes_sessions_on_accessor_drop) -{ - const uint64_t cap = 4; - const int zones = 20; - - FakeWorldtreeDaemon fake((tmpDir / "w.sock").string()); - Ctx ctx; - EvalState & state = ctx.wire(fake.path(), cap); - - Hash sha(HashAlgorithm::SHA1); - - std::vector> handles; - for (int i = 0; i < zones; ++i) - handles.push_back(acquire(state, sha, "//areas/z" + std::to_string(i))); - - // Nothing is released while the handles are still held. - EXPECT_TRUE(fake.closedSessions().empty()); - - handles.clear(); // drop every accessor session - - // Every opened session was closed exactly once (ids 1..M). - std::set expected; - for (int i = 1; i <= zones; ++i) - expected.insert(static_cast(i)); - EXPECT_EQ(fake.closedSessions(), expected); -} - -} // namespace nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b803f1243..6b401465b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -35,6 +35,7 @@ #include "parser-tab.hh" #include +#include #include #include #include @@ -44,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -542,29 +544,19 @@ static std::string sanitizeZoneNameForStore(std::string_view zonePath) // ============================================================================ // Worldtree daemon integration (design §5.1 / §5.1a) // -// When `tectonix-worldtree-socket` is set, three tectonix couplings move off the -// libgit2 repo+checkout walk and onto O(changes) RPCs against a worldtreed workspace: +// When `tectonix-worldtree-socket` is set, mutable-checkout metadata moves off the +// libgit2 repo+checkout walk and onto O(changes) RPCs against the bound workspace: // * getTectonixDirtyZones() -> `dirty_zones` — the tracked-dirty set; // * getWorldTreeSha() -> `zone_tree_shas` — the working-tree subtree oid // (the committed oid when clean, the synthesized frontier oid when dirty; // the zone's build-cache key); -// * getZoneStorePath() -> source bytes, split by the §5.1a dichotomy: -// - the workspace's *own* state (a checkout/mount path is set) is materialized, -// so its merged working tree is read as plain local files — never the socket; -// - with no local checkout, a foreign SHA is *always clean*, so clean committed -// content is streamed over the socket by a `WorldtreeSourceAccessor` -// (one `read_tree` prefetch + lazy `read_blobs`). +// * getZoneStorePath() reads source bytes from FUSE in both regimes: `root` for the +// mutable workspace and `tecnix//` for immutable history. // Fail-loud contract: when the socket is SET, the daemon is the sole source of truth — a // worldtree sandbox has no git repo to fall back to. A daemon that is unreachable, or that // refuses/errors a request, is a hard failure (the error propagates), never a silent // downgrade. libgit2 / the checkout walk are reached ONLY when the socket is UNSET (plain -// local, non-worldtree eval). This is the load-bearing invariant behind the two modes. -// -// One `read_tree` ships the whole zone subtree skeleton; this bounds the descent well -// past any real zone depth (the daemon stops at the true leaves regardless), so the -// prefetch is never silently truncated. -static constexpr uint32_t WORLDTREE_PREFETCH_DEPTH = 1u << 20; - +// local, non-worldtree eval). Historical reads create no daemon connection. /** Reinterpret a 20-byte worldtree object id as a Nix SHA-1 Hash. */ static Hash oidToHash(const worldtree::Oid & oid) { @@ -575,10 +567,9 @@ static Hash oidToHash(const worldtree::Oid & oid) } /** - * A single connection to a worldtree daemon bound to one workspace. The transport is a - * synchronous, one-call-at-a-time client, so every verb is serialized under `mutex`; - * cross-zone parallelism comes from each zone's content accessor holding its *own* - * `WorldtreeConn` (see `getZoneStorePath`), not from concurrency within one connection. + * The scoped socket is only the mutable root-checkout control plane. Historical + * committed source is ordinary filesystem input beneath + * /mnt/worldtree/tecnix//. */ struct WorldtreeConn { @@ -586,75 +577,13 @@ struct WorldtreeConn std::mutex mutex; worldtree::Client client; - // Session state (design §5.1a/§6.1). A conn is either *bound* (ws is the sandbox's own - // workspace, set at construction — the `tec ` control conn) or a *session* (ws is an - // ephemeral RO workspace this conn opened via `open_ro`, pinned at a SHA — the `tec --sha` - // path). `ownsSession` is set iff we opened one, so the destructor releases exactly what - // we created (§6.1: "the new workspace is owned by the calling sandbox"). - bool ownsSession = false; - std::string sessionManifest_; // in-band `.meta/manifest.json` at the pinned SHA - uint64_t basePin = 0; // generation the session refcounts (observability) - WorldtreeConn(worldtree::Client && client, uint64_t ws) : ws(ws) , client(std::move(client)) { } - ~WorldtreeConn() - { - // RAII release of an ephemeral RO session: the daemon also drops it on disconnect, so - // this is the prompt, well-behaved path. Best-effort — a destructor must not throw, - // and a lost connection (the very case where close would fail) already frees it - // daemon-side. - if (ownsSession) { - try { - std::lock_guard lock(mutex); - client.closeRo(ws); - } catch (...) { - } - } - } - - /** - * Open an ephemeral RO session pinned at `baseSha`, confined to a zone cone (empty cone ⇒ - * full visibility). Replaces `ws` with the returned session id and caches the in-band - * manifest; every subsequent read verb on this conn then addresses the session. One - * session per conn (the conn *is* the session once opened). - */ - void openSession( - const Hash & baseSha, - const std::string & visMode, - const std::vector & zoneIds, - const std::vector & zonePaths) - { - worldtree::Oid oid{}; - assert(baseSha.hashSize == oid.size()); - std::memcpy(oid.data(), baseSha.hash, oid.size()); - std::lock_guard lock(mutex); - auto session = client.openRo(oid, visMode, zoneIds, zonePaths); - ws = session.ws; - sessionManifest_ = std::move(session.manifest); - basePin = session.basePin; - ownsSession = true; - } - - /** The session's in-band manifest, or nullopt when this conn is bound (not a session). */ - std::optional sessionManifest() const - { - if (!ownsSession) - return std::nullopt; - return sessionManifest_; - } - - /** World paths (`//zone`) of zones with tracked working-copy changes. */ - std::vector dirtyZones() - { - std::lock_guard lock(mutex); - return client.dirtyZones(ws); - } - - /** The dirty set with each zone's changed files (for full `ZoneDirtyInfo`). */ + /** The dirty set with each zone's changed files (for full ZoneDirtyInfo). */ std::vector dirtyZoneEntries() { std::lock_guard lock(mutex); @@ -664,7 +593,6 @@ struct WorldtreeConn /** One zone's working-tree subtree oid, or nullopt when absent or out of scope. */ std::optional zoneTreeSha(std::string_view worldPath) { - // `zone_tree_shas` expects World paths (`//zone`); normalize to that form. std::string wp = hasPrefix(worldPath, "//") ? std::string(worldPath) : "//" + std::string(worldPath); std::lock_guard lock(mutex); auto resp = client.zoneTreeShas(ws, {wp}); @@ -672,315 +600,97 @@ struct WorldtreeConn return std::nullopt; return oidToHash(*resp.front().treeSha); } - - /** Prefetch a committed subtree skeleton; entry paths are relative to `rel`. */ - std::vector readTree(std::string_view rel, uint32_t depth) - { - std::lock_guard lock(mutex); - return client.readTree(ws, std::string(rel), depth); - } - - /** Decoded bytes of one committed blob by oid, or nullopt when absent. */ - std::optional readBlob(const worldtree::Oid & oid) - { - std::lock_guard lock(mutex); - auto blobs = client.readBlobs(ws, {oid}); - if (blobs.empty()) - return std::nullopt; - return blobs.front().content; - } - - // ---- Multiplexed-session API (the bounded connection pool) ---- - // - // The single-session methods above address this conn's own `ws` (a bound workspace, or - // the one session it opened via `openSession`). The methods below instead carry an - // explicit session `ws`, so one physical connection can host many independent per-zone - // ephemeral RO sessions at once (the daemon's owned-ephemeral set is per-connection). - // Each still serializes on `mutex` like every other verb; cross-session parallelism - // comes from the *pool* holding several connections, not from concurrency within one. - - /** - * Open a zone-scoped ephemeral RO session hosted on this connection and return its id, - * *without* touching `ws`/`ownsSession` (unlike `openSession`, which rebinds this conn - * to a single session). The caller owns the returned session and must `closeSessionOn` - * it; any left open at connection teardown are released daemon-side on disconnect. - */ - uint64_t openScopedSession(const Hash & baseSha, const std::string & worldPath) - { - worldtree::Oid oid{}; - assert(baseSha.hashSize == oid.size()); - std::memcpy(oid.data(), baseSha.hash, oid.size()); - std::lock_guard lock(mutex); - return client.openRo(oid, "scoped", {}, {worldPath}).ws; - } - - /** Release a hosted session opened on this connection (best-effort; ownership-gated). */ - void closeSessionOn(uint64_t sessionWs) noexcept - { - try { - std::lock_guard lock(mutex); - client.closeRo(sessionWs); - } catch (...) { - } - } - - /** `read_tree` on a hosted session; entry paths are relative to `rel`. */ - std::vector readTreeOn(uint64_t sessionWs, std::string_view rel, uint32_t depth) - { - std::lock_guard lock(mutex); - return client.readTree(sessionWs, std::string(rel), depth); - } - - /** Decoded bytes of one committed blob on a hosted session, or nullopt when absent. */ - std::optional readBlobOn(uint64_t sessionWs, const worldtree::Oid & oid) - { - std::lock_guard lock(mutex); - auto blobs = client.readBlobs(sessionWs, {oid}); - if (blobs.empty()) - return std::nullopt; - return blobs.front().content; - } }; -/** - * A single per-zone ephemeral RO session hosted on a pooled worldtree connection. The - * session (`ws`, scoped to one zone's cone) is owned by this handle and released - * (`close_ro` on the hosting connection) when the handle drops — i.e. when the zone's - * `WorldtreeSourceAccessor`, its sole owner, is torn down. `carrier` is shared with the - * other sessions multiplexed over the same connection (the bounded pool), so many zones - * read concurrently over few connections instead of one connection per zone. - */ -struct WorldtreeZoneSession -{ - std::shared_ptr carrier; - uint64_t ws; - - WorldtreeZoneSession(std::shared_ptr carrier, uint64_t ws) - : carrier(std::move(carrier)) - , ws(ws) - { - } - - WorldtreeZoneSession(const WorldtreeZoneSession &) = delete; - WorldtreeZoneSession & operator=(const WorldtreeZoneSession &) = delete; +static constexpr std::string_view WORLDTREE_TREE_OID_XATTR = "user.worldtree.tree-oid"; - ~WorldtreeZoneSession() - { - if (carrier) - carrier->closeSessionOn(ws); - } - - /** Prefetch this zone's committed subtree skeleton; paths are relative to `rel`. */ - std::vector readTree(std::string_view rel, uint32_t depth) - { - return carrier->readTreeOn(ws, rel, depth); - } - - /** Decoded bytes of one committed blob in this zone's session, or nullopt when absent. */ - std::optional readBlob(const worldtree::Oid & oid) - { - return carrier->readBlobOn(ws, oid); - } -}; - -/** - * The §5.1a source accessor: a `GitSourceAccessor`-shaped view of a *clean committed* - * zone subtree served over `worldtree.sock`, with no per-SHA delegate FUSE mount. One - * `read_tree` prefetch ships the whole skeleton (path/mode/oid/size, no content), after - * which `maybeLstat`/`readDirectory`/`readLink` are answered locally with zero - * round-trips; `readFile` fetches the blob by oid on first touch and caches it (blobs - * are immutable, so the cache never staleness-checks). Used only when there is no local - * checkout — the `tec --sha` path, which is clean by construction. - * - * Confinement (design §10.1, the load-bearing constraint): `session` is an ephemeral RO - * session opened `scoped([zone])`, so the daemon — the authority — refuses any path read - * outside the zone cone. This accessor is *also* rooted at the zone (all reads go through - * `zoneRel`), and its constructor rejects a `zoneRel` that could escape, as - * defense-in-depth. The two together guarantee a zone build cannot traverse out of its root. - * The session is hosted on a pooled connection shared with other zones' sessions - * (`acquireWorldtreeZoneSession`); it is released when this accessor, its sole owner, drops. - */ -struct WorldtreeSourceAccessor : SourceAccessor +static std::filesystem::path worldtreeRevisionRoot(const EvalSettings & settings) { - std::shared_ptr session; - std::string zoneRel; // ws-relative, no leading `//` (e.g. "areas/tools/dev") - - struct Node - { - uint32_t mode = 0; - worldtree::Oid oid{}; - uint64_t size = 0; - bool isDir = false; - }; - - std::once_flag prefetchFlag; - // Skeleton keyed by zone-relative path ("" is the zone root). `read_tree` returns - // entry paths relative to the requested root (= the zone), so no prefix-stripping. - std::map nodes; - std::map> dirChildren; - - // Immutable blob content cache, by oid. Per-accessor; a host-shared content-addressed - // cache across zones/builds (§5.1a) is a future optimization layered on this. - std::mutex blobMutex; - std::map blobCache; - - WorldtreeSourceAccessor(std::shared_ptr session, std::string zoneRel, const Hash & treeSha) - : session(std::move(session)) - , zoneRel(std::move(zoneRel)) - { - // Defense-in-depth escape guard: the zone root we prefetch from must be a plain - // ws-relative path. A leading `/` or any `..` component would let a read reach - // outside the zone cone (the daemon would still refuse it, but we never form the - // request). This is the C++ half of the confinement invariant. - if (hasPrefix(this->zoneRel, "/")) - throw Error("worldtree: zone root '%s' must be workspace-relative", this->zoneRel); - for (auto & c : tokenizeString>(this->zoneRel, "/")) - if (c == ".." || c == ".") - throw Error("worldtree: zone root '%s' contains an illegal path component", this->zoneRel); - - // A stable, content-addressed eval-cache key (design §5.1a): the committed subtree - // oid uniquely identifies this accessor's bytes, so two evaluations of the same zone - // at the same content share the fetch cache — and, unlike a checkout-derived default, - // it is correct for socket-served content that has no local path. - fingerprint = "worldtree:" + treeSha.gitRev(); - } - - static std::string key(const CanonPath & path) - { - return path.isRoot() ? std::string() : std::string(path.rel()); - } - - static Type typeOfMode(uint32_t mode, bool isDir) - { - if (isDir) - return tDirectory; - if ((mode & 0170000) == 0120000) - return tSymlink; - return tRegular; - } - - void prefetch() - { - std::call_once(prefetchFlag, [this]() { - auto entries = session->readTree(zoneRel, WORLDTREE_PREFETCH_DEPTH); - nodes[""] = Node{.mode = 0040000, .isDir = true}; // the zone root itself - for (auto & e : entries) { - bool isDir = (e.mode & 0170000) == 0040000; - nodes[e.path] = Node{.mode = e.mode, .oid = e.oid, .size = e.size, .isDir = isDir}; - auto slash = e.path.rfind('/'); - std::string parent = slash == std::string::npos ? "" : e.path.substr(0, slash); - std::string base = slash == std::string::npos ? e.path : e.path.substr(slash + 1); - dirChildren[parent].insert(base); - } - }); - } - - std::optional maybeLstat(const CanonPath & path) override - { - prefetch(); - auto it = nodes.find(key(path)); - if (it == nodes.end()) - return std::nullopt; - const auto & n = it->second; - Stat st; - st.type = typeOfMode(n.mode, n.isDir); - if (st.type == tRegular) { - st.fileSize = n.size; - st.isExecutable = (n.mode & 0111) != 0; - } - return st; - } + auto revision = Hash::parseNonSRIUnprefixed(settings.tectonixGitSha.get(), HashAlgorithm::SHA1); + return std::filesystem::path(settings.tectonixWorldtreeMount.get()) / "tecnix" / revision.gitRev(); +} - DirEntries readDirectory(const CanonPath & path) override - { - prefetch(); - DirEntries entries; - auto it = dirChildren.find(key(path)); - if (it == dirChildren.end()) - return entries; - auto dir = key(path); - for (const auto & base : it->second) { - auto childKey = dir.empty() ? base : dir + "/" + base; - std::optional type; - if (auto n = nodes.find(childKey); n != nodes.end()) - type = typeOfMode(n->second.mode, n->second.isDir); - entries[base] = type; - } - return entries; - } +static std::string requireWorldtreeZoneId(const std::string & id) +{ + auto isLowerHex = [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); }; + if (id.size() != 8 || !id.starts_with("W-") || !std::ranges::all_of(std::string_view(id).substr(2), isLowerHex)) + throw Error("worldtree: invalid zone id '%s' in manifest", id); + return id; +} - // Override the streaming variant. The base's `std::string readFile(path)` is a - // non-virtual convenience wrapper (it funnels through this one), so overriding *it* - // does not compile — every accessor implements the Sink form instead. Blob bytes come - // from the per-accessor oid cache, else a single socket read. `blobCache` is a - // node-based `std::map`, so a reference into it stays valid after the lock is dropped - // (entries are only inserted, never erased) — letting the potentially-large sink write - // run lock-free, as the prior string-returning version did for its network read. - void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override - { - prefetch(); - auto it = nodes.find(key(path)); - if (it == nodes.end() || it->second.isDir) - throw Error("worldtree: '%s' is not a readable file in zone '%s'", showPath(path), zoneRel); - const auto & oid = it->second.oid; +/** Find the manifest zone containing worldPath and return (zone id, path within zone). */ +static std::pair +worldtreeZoneLocation(const nlohmann::json & manifest, std::string_view worldPath) +{ + auto clean = normalizeZonePath(worldPath); + while (!clean.empty() && clean.front() == '/') + clean.erase(clean.begin()); + while (!clean.empty() && clean.back() == '/') + clean.pop_back(); + for (auto & component : tokenizeString>(clean, "/")) + if (component.empty() || component == "." || component == "..") + throw Error("invalid world path '%s'", worldPath); - const std::string * content = nullptr; - { - std::lock_guard lock(blobMutex); - if (auto c = blobCache.find(oid); c != blobCache.end()) - content = &c->second; - } - if (!content) { - auto fetched = session->readBlob(oid); // network read, no lock held - if (!fetched) - throw Error("worldtree: content for '%s' is missing from the workspace", showPath(path)); - std::lock_guard lock(blobMutex); - content = &blobCache.try_emplace(oid, std::move(*fetched)).first->second; + const nlohmann::json * best = nullptr; + std::string bestPath; + for (auto & [candidateWorldPath, value] : manifest.items()) { + auto candidate = normalizeZonePath(candidateWorldPath); + bool contains = + clean == candidate + || (clean.size() > candidate.size() && hasPrefix(clean, candidate) && clean[candidate.size()] == '/'); + if (contains && candidate.size() > bestPath.size()) { + best = &value; + bestPath = std::move(candidate); } - - sizeCallback(content->size()); - sink(*content); } + if (!best || !best->is_object() || !best->contains("id") || !best->at("id").is_string()) + throw Error("worldtree: path '%s' is not contained by a visible World zone", worldPath); - std::string readLink(const CanonPath & path) override - { - // A symlink's target is stored as its blob content. Qualify the call: declaring - // the Sink-based `readFile` override above hides the base's non-virtual - // `std::string readFile(path)` convenience wrapper in this scope (C++ name - // hiding), so name it explicitly — it dispatches back through our override and - // returns the blob bytes as a string. - return SourceAccessor::readFile(path); - } + auto id = requireWorldtreeZoneId(best->at("id").get()); + auto relative = clean.size() == bestPath.size() ? std::string() : clean.substr(bestPath.size() + 1); + return {std::move(id), std::move(relative)}; +} - std::optional getLastModified() override - { - // The wire carries no timestamps — committed blobs are content-addressed, not dated. - // Return an explicit "unknown" (matching the base default) rather than silently - // inheriting a value that would be meaningless for socket-served content; the - // content-addressed `fingerprint` above is the correct cache key here, not mtime. - return std::nullopt; - } -}; +static Hash readWorldtreeTreeOid(const std::filesystem::path & path) +{ + std::array value{}; +#ifdef __APPLE__ + auto size = ::getxattr(path.c_str(), WORLDTREE_TREE_OID_XATTR.data(), value.data(), value.size(), 0, 0); +#else + auto size = ::getxattr(path.c_str(), WORLDTREE_TREE_OID_XATTR.data(), value.data(), value.size()); +#endif + if (size < 0) + throw Error("worldtree: cannot read tree identity for '%s': %s", path.string(), std::strerror(errno)); + if (size != static_cast(value.size())) + throw Error("worldtree: invalid tree identity on '%s'", path.string()); + return Hash::parseNonSRIUnprefixed(std::string(value.data(), value.size()), HashAlgorithm::SHA1); +} Hash EvalState::getWorldTreeSha(std::string_view worldPath) const { - // Worldtree mode (design §5.1a): the daemon is authoritative for a world path's - // working-tree oid (committed when clean, the synthesized frontier oid when dirty). - // With the socket set the daemon is the sole source of truth — a worldtree sandbox has - // no git repo to fall back to. An absent oid is a *normal* daemon verdict for a path - // that is missing or outside this workspace's visibility scope (the handler encodes it - // as an empty tree_sha in a successful response, which maps to nullopt here), so it must - // be treated as terminal: falling through to the libgit2 walk would silently serve a - // zone the daemon deliberately hid, or committed content at a different generation than - // the pinned session — the exact silent downgrade decision #1 forbids. libgit2 (below) - // is reachable only when no socket is configured. Mirrors getZoneStorePath / - // getManifestContent. - if (auto control = worldtreeControlConn()) { + // The mutable root checkout needs its synthesized working-tree oid from the control + // plane. A historical evaluation is already pinned by its FUSE path and reads the + // exact committed tree oid from that directory's synthetic xattr instead. + if (isTectonixSourceAvailable()) { + auto control = worldtreeControlConn(); + if (!control) + goto local_git; if (auto sha = control->zoneTreeSha(worldPath)) return *sha; throw Error("worldtree: world path '%s' is absent or outside this workspace's visibility scope", worldPath); } + if (!settings.tectonixWorldtreeSocket.get().empty()) { + auto revision = worldtreeRevisionRoot(settings); + if (normalizeZonePath(worldPath).empty()) + return readWorldtreeTreeOid(revision); + auto [zoneId, relative] = worldtreeZoneLocation(getManifestJson(), worldPath); + auto path = revision / zoneId; + if (!relative.empty()) + path /= relative; + return readWorldtreeTreeOid(path); + } +local_git: auto path = normalizeZonePath(worldPath); // Check cache first @@ -1086,16 +796,25 @@ const std::set & EvalState::getTectonixSparseCheckoutRoots() const const std::map & EvalState::getTectonixDirtyZones() const { std::call_once(tectonixDirtyZonesFlag, [this]() { + // A historical FUSE view is immutable by construction. Preserve the usual full + // manifest-shaped result (every visible zone present and clean) without opening a + // control connection or manufacturing an ephemeral daemon workspace. + if (!isTectonixSourceAvailable() && !settings.tectonixWorldtreeSocket.get().empty()) { + for (auto & [zonePath, value] : getManifestJson().items()) + if (value.is_object() && value.contains("id") && value.at("id").is_string()) + tectonixDirtyZones[zonePath] = {}; + return; + } + // Worldtree mode: the daemon is authoritative for the tracked-dirty set (design // §5.1), derived from the materialization frontier in O(changes) — no O(working-tree) // `git status` scan. Reconstruct a *full* ZoneDirtyInfo so every consumer (notably // the `__unsafeTectonixInternalDirtyZones` primop) sees every manifest zone with an // accurate flag, matching the libgit2 path's shape: - // (1) init every manifest zone clean — the manifest is served over the socket in - // `tec --sha` mode and read from the checkout in `tec ` mode (see - // getManifestContent); either way it enumerates the full zone set; - // (2) overlay the daemon's per-zone dirty files (empty for a clean --sha session, - // the real staged ∪ unstaged set for a bound sandbox workspace). + // (1) init every manifest zone clean — the immutable FUSE view supplies it in + // historical mode and the checkout supplies it in mutable mode (see + // getManifestContent); either way it enumerates the full visible zone set; + // (2) overlay the daemon's per-zone dirty files for the bound mutable workspace. if (auto control = worldtreeControlConn()) { const nlohmann::json * manifest; try { @@ -1244,6 +963,14 @@ const std::map & EvalState::getTectonixDi // Path to the tectonix manifest file within the world repository static constexpr std::string_view TECTONIX_MANIFEST_PATH = "/.meta/manifest.json"; +[[noreturn]] static void throwHistoricalWorldManifestError(const EvalSettings & settings, std::string_view detail) +{ + throw Error( + "worldtree: historical World manifest '.meta/manifest.json' for commit '%s' is missing or malformed: %s", + settings.tectonixGitSha.get(), + detail); +} + const std::string & EvalState::getManifestContent() const { // Cached for the lifetime of evaluation. This is intentional: evaluation is @@ -1264,25 +991,21 @@ const std::string & EvalState::getManifestContent() const } } - // Mode B (`tec --sha`, no checkout): the manifest at the pinned SHA is delivered - // in-band by the daemon in the control session's `open_ro` response — no libgit2, no - // path traversal to `.meta`. Fail-loud: with the socket set this is the only source. - if (auto control = worldtreeControlConn()) { - if (auto m = control->sessionManifest()) { - tectonixManifestContent = std::move(*m); - debug("loaded manifest over the worldtree socket at the pinned SHA"); - return; + // Mode B (`tec --ref`, no checkout): manifest metadata is an ordinary immutable + // file in the FUSE projection. W-000000 is the reserved manifest pseudo-zone; it + // follows the same workspace visibility as the root checkout and needs no socket. + if (!settings.tectonixWorldtreeSocket.get().empty()) { + auto manifestPath = worldtreeRevisionRoot(settings) / "W-000000" / "manifest.json"; + std::error_code ec; + if (!std::filesystem::is_regular_file(manifestPath, ec)) + throwHistoricalWorldManifestError(settings, ec ? ec.message() : "file does not exist"); + try { + tectonixManifestContent = readFile(manifestPath); + } catch (const Error & e) { + throwHistoricalWorldManifestError(settings, e.what()); } - // The socket is set but neither source produced a manifest: Mode A had no - // checkout file on disk (above) AND this control conn is bound, not an RO - // session (so it carries no in-band manifest). With the socket set the daemon - // is the sole source of truth (decisions #1/#6) — fail loud rather than - // silently reading stale/wrong committed content from libgit2. This also keeps - // getTectonixDirtyZones' fail-loud catch intact (it relies on this throwing). - throw Error( - "worldtree: manifest.json is unavailable (socket is set: no checkout " - "manifest on disk and the control connection is bound, not an RO session) " - "— refusing to fall back to libgit2"); + debug("loaded manifest from immutable worldtree view: %s", manifestPath.string()); + return; } // Socket unset (plain local eval): read the committed manifest via libgit2. @@ -1299,50 +1022,47 @@ const std::string & EvalState::getManifestContent() const const nlohmann::json & EvalState::getManifestJson() const { std::call_once(tectonixManifestJsonFlag, [this]() { - tectonixManifestJson = std::make_unique(nlohmann::json::parse(getManifestContent())); + try { + tectonixManifestJson = std::make_unique(nlohmann::json::parse(getManifestContent())); + } catch (const nlohmann::json::parse_error & e) { + if (!settings.tectonixWorldtreeSocket.get().empty() && !isTectonixSourceAvailable()) + throwHistoricalWorldManifestError(settings, e.what()); + throw; + } }); return *tectonixManifestJson; } StorePath EvalState::getZoneStorePath(std::string_view zonePath) { - // Worldtree mode (design §5.1a): the daemon owns the zone's identity and its source. - // The working-tree oid is the dedup/build key (clean → committed, dirty → - // synthesized). Source bytes follow the §5.1a dichotomy below. This returns before - // the libgit2 dirty/clean split, so the checkout overlay is never built in this mode. - if (auto control = worldtreeControlConn()) { - auto treeSha = control->zoneTreeSha(zonePath); - if (!treeSha) - throw Error("worldtree: zone '%s' is absent or outside this workspace's visibility scope", zonePath); - - ref accessor = [&]() -> ref { - if (isTectonixSourceAvailable()) { - // Mode A (`tec `): the sandbox's own materialized mount already presents - // the merged working tree (base ⊕ uncommitted edits), so read it as local - // files — at full speed, never over the socket (§5.1a). `treeSha` here is the - // bound workspace's dirty frontier oid, the correct build key. - auto fullPath = - std::filesystem::path(settings.tectonixCheckoutPath.get()) / normalizeZonePath(zonePath); - if (!std::filesystem::exists(fullPath)) - throw Error("worldtree: zone '%s' is not materialized at '%s'", zonePath, fullPath.string()); - return makeFSSourceAccessor(fullPath); - } - // Mode B (`tec --sha`): no checkout — a foreign SHA is clean by construction, so - // serve committed content over the socket via an ephemeral RO session **scoped to - // this zone's cone** (design §10.1 confinement), pinned at the target SHA; the - // daemon then refuses any read outside the zone root. The session is hosted on a - // connection drawn from a bounded pool (`tectonix-worldtree-max-connections`), so a - // broad evaluation multiplexes many zones over few connections instead of opening - // one per zone (which could exceed the daemon's per-listener connection cap and - // surface as `worldtree: read(): Connection reset by peer`). The session releases - // when the accessor (its sole owner) drops. Fail-loud: an unreachable daemon throws. - auto sha = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); - auto worldPath = hasPrefix(zonePath, "//") ? std::string(zonePath) : "//" + std::string(zonePath); - auto session = acquireWorldtreeZoneSession(sha, worldPath); - return make_ref(session, normalizeZonePath(zonePath), *treeSha); - }(); - - return worldtreeMountAccessor(*treeSha, zonePath, accessor); + // A worldtree sandbox has two source regimes but only one filesystem accessor: + // the mutable root checkout path for ordinary evaluation, or the immutable + // commit/zone path for --ref. Only the former needs control RPCs for its dirty + // frontier identity. + if (!settings.tectonixWorldtreeSocket.get().empty()) { + if (isTectonixSourceAvailable()) { + auto control = worldtreeControlConn(); + if (!control) + throw Error("worldtree: mutable checkout has no control connection"); + auto treeSha = control->zoneTreeSha(zonePath); + if (!treeSha) + throw Error("worldtree: zone '%s' is absent or outside this workspace's visibility scope", zonePath); + auto fullPath = std::filesystem::path(settings.tectonixCheckoutPath.get()) / normalizeZonePath(zonePath); + if (!std::filesystem::is_directory(fullPath)) + throw Error("worldtree: zone '%s' is not materialized at '%s'", zonePath, fullPath.string()); + return worldtreeMountAccessor(*treeSha, zonePath, makeFSSourceAccessor(fullPath)); + } + + auto manifestIt = getManifestJson().find(std::string(zonePath)); + if (manifestIt == getManifestJson().end() || !manifestIt->is_object() || !manifestIt->contains("id") + || !manifestIt->at("id").is_string()) + throw Error("worldtree: zone '%s' is absent from the visible manifest", zonePath); + auto zoneId = requireWorldtreeZoneId(manifestIt->at("id").get()); + auto fullPath = worldtreeRevisionRoot(settings) / zoneId; + if (!std::filesystem::is_directory(fullPath)) + throw Error("worldtree: immutable zone '%s' is unavailable at '%s'", zonePath, fullPath.string()); + auto treeSha = readWorldtreeTreeOid(fullPath); + return worldtreeMountAccessor(treeSha, zonePath, makeFSSourceAccessor(fullPath)); } // Check dirty status using original zonePath (with // prefix) since @@ -1451,64 +1171,19 @@ std::shared_ptr EvalState::connectWorldtree() const // Fail-loud (the load-bearing invariant): with the socket SET there is no git repo to // fall back to, so an unreachable daemon is a hard error — let `Client::connect`'s // `ProtocolError` propagate rather than silently degrading to libgit2 (which would read - // the wrong content, or none). A bare (bound) connection; a session is opened by the - // caller when one is needed. + // the wrong content, or none). auto ws = settings.tectonixWorldtreeWorkspace.get(); return std::make_shared(worldtree::Client::connect(socketPath), ws); } -std::shared_ptr -EvalState::acquireWorldtreeZoneSession(const Hash & sha, const std::string & worldPath) const -{ - // Draw a connection round-robin from a bounded pool and host this zone's scoped session - // on it, so a broad evaluation multiplexes many per-zone sessions over at most - // `tectonix-worldtree-max-connections` connections rather than one connection per zone. - // The old one-connection-per-zone shape could exceed the daemon's per-listener connection - // cap on a wide changeset, which the client surfaces as - // `worldtree: read(): Connection reset by peer`. - std::shared_ptr carrier; - { - std::lock_guard lock(worldtreePoolMutex_); - uint64_t cap = std::max(1, settings.tectonixWorldtreeMaxConnections.get()); - if (worldtreePool_.size() < cap) { - // Grow lazily: open a new connection only while under the cap, so a small - // evaluation never pays for connections it will not use. Fail-loud — a fresh - // connection throws if the daemon is unreachable (there is no libgit2 fallback), - // and returns null only when the socket is unset, which Mode B never is. - auto conn = connectWorldtree(); - if (!conn) - throw Error("worldtree: socket is not configured while opening zone '%s'", worldPath); - worldtreePool_.push_back(conn); - carrier = std::move(conn); - } else { - carrier = worldtreePool_[worldtreePoolNext_ % worldtreePool_.size()]; - worldtreePoolNext_++; - } - } - // openScopedSession takes the connection's own lock; done outside the pool lock so - // session opens on different connections proceed concurrently. - auto sessionWs = carrier->openScopedSession(sha, worldPath); - return std::make_shared(std::move(carrier), sessionWs); -} - std::shared_ptr EvalState::worldtreeControlConn() const { - std::call_once(worldtreeControlConnFlag, [this]() { - auto conn = connectWorldtree(); - // Mode B (`tec --sha`, no local checkout): the control connection hosts a - // *full-visibility* ephemeral RO session pinned at the target SHA. It answers the - // host-level control-plane reads — the in-band manifest (getManifestContent), the - // dirty set (getTectonixDirtyZones, empty for a clean session), and zone tree oids - // (getWorldTreeSha/getZoneStorePath). Full visibility is correct here: confinement - // (§10.1) is a per-zone *source*-read hardening, not a control-plane boundary. In - // Mode A (`tec `) the conn stays bound to the sandbox's own workspace (no - // session) and these reads run against it directly. - if (conn && !isTectonixSourceAvailable()) { - auto sha = Hash::parseNonSRIUnprefixed(requireTectonixGitSha(), HashAlgorithm::SHA1); - conn->openSession(sha, "full", {}, {}); - } - worldtreeControlConn_ = conn; - }); + // Historical --ref evaluations are filesystem-only. Returning null here is not a + // libgit2 fallback: their callers branch on worldtree mode before consulting this + // control seam. + if (!isTectonixSourceAvailable()) + return nullptr; + std::call_once(worldtreeControlConnFlag, [this]() { worldtreeControlConn_ = connectWorldtree(); }); return worldtreeControlConn_; } diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 7d999065a..a7e65b383 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -551,19 +551,21 @@ struct EvalSettings : Config R"( Path to a worldtree daemon (`worldtreed`) control socket. - When set, the tectonix builtins read the working copy from the daemon over - its socket instead of walking the git repository and checkout with libgit2: - zone tree shas, the dirty-zone set, and zone source content are all served by - O(changes) RPCs against the workspace named by `tectonix-worldtree-workspace`. + When set, the tectonix builtins read mutable checkout metadata from the daemon + instead of walking the git repository with libgit2: working tree shas and the + dirty-zone set are O(changes) RPCs against the workspace named by + `tectonix-worldtree-workspace`. Source bytes are read from the worldtree FUSE + projection: the current checkout path or the immutable historical path beneath + `tectonix-worldtree-mount`. This replaces the two libgit2 couplings — the per-zone tree-sha walk and the O(working-tree) `git status` dirty scan — that dominate evaluation in a large checkout. - Empty (the default) keeps the libgit2 path. When set, the daemon is the sole - source of truth (a worldtree sandbox has no git repo to fall back to): an - unreachable daemon, or one that refuses a request, is a hard failure — evaluation - aborts rather than silently reading stale or wrong content from libgit2. Set this - only where a daemon is actually serving the source. + Empty (the default) keeps the libgit2 path. When set, a mutable checkout has no + git fallback: an unreachable daemon, or one that refuses a control request, is a + hard failure. Historical evaluation does not connect to the socket; a missing or + invalid immutable FUSE path is likewise a hard failure. Set this only where + worldtree is actually serving the source. )"}; Setting tectonixWorldtreeWorkspace{ @@ -577,29 +579,18 @@ struct EvalSettings : Config evaluation. Ignored when `tectonix-worldtree-socket` is empty. )"}; - Setting tectonixWorldtreeMaxConnections{ + Setting tectonixWorldtreeMount{ this, - 64, - "tectonix-worldtree-max-connections", + "/mnt/worldtree", + "tectonix-worldtree-mount", R"( - Maximum number of physical control-socket connections tectonix opens to a - worldtree daemon for per-zone source reads (the `tectonix-worldtree-socket` - path). - - A `tec --sha` / worldtree-sandbox evaluation serves each clean zone's committed - source over its own ephemeral read-only session. A broad evaluation — e.g. the - Tartarus planning bundle across a large changeset — touches many zones at once; - opening a separate socket connection per zone can exceed the daemon's per-listener - connection cap, which the client then sees as - `worldtree: read(): Connection reset by peer`. To bound this, zone sessions are - multiplexed over a pool of at most this many connections (the daemon's - owned-ephemeral session set is per-connection, so one connection can host many - sessions); reads that land on the same connection serialize on it. - - Must stay below the daemon's scoped-listener `max_connections` (512). The default - (64) comfortably exceeds typical core counts, so per-core read parallelism is - preserved while the live connection count stays far under the cap. A value of 0 is - treated as 1. Ignored when `tectonix-worldtree-socket` is empty. + Root of the worldtree workspace projection inside a sandbox. Historical, + immutable zone views live at `tecnix//` beneath this + directory; the current mutable checkout lives at `root`. + + This setting is consulted only when `tectonix-worldtree-socket` is set. The + socket remains the control plane for the mutable root checkout; committed + Tecnix source bytes and manifest metadata are read directly from this filesystem. )"}; }; diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 46956f89c..a32a7c865 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -56,9 +56,6 @@ class EvalState; /** A connection to a worldtree daemon, defined in eval.cc (it owns the C++ worldtree * client). Held by pointer here so the worldtree client header stays out of eval.hh. */ struct WorldtreeConn; -/** A per-zone ephemeral RO session hosted on a pooled WorldtreeConn, defined in eval.cc. - * Forward-declared for the same reason as WorldtreeConn. */ -struct WorldtreeZoneSession; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; @@ -564,7 +561,8 @@ private: /** * Lazily-connected worldtree daemon control connection (zone tree shas + the dirty - * set), or null only when `tectonix-worldtree-socket` is unset (plain local eval). + * set), or null when the socket is unset or the evaluation targets an immutable + * historical FUSE view rather than the mutable root checkout. * With the socket set an unreachable daemon THROWS rather than yielding null — there * is no null-on-unreachable and no libgit2 fallback (fail-loud; see the * `tectonix-worldtree-socket` setting doc). Connected at most once; thread-safe via @@ -573,33 +571,6 @@ private: mutable std::once_flag worldtreeControlConnFlag; mutable std::shared_ptr worldtreeControlConn_; - /** - * Bounded pool of worldtree control-socket connections that host the per-zone - * ephemeral RO sessions of the socket-served source path (Mode B, `tec --sha`). Each - * connection can own many sessions (the daemon's owned-ephemeral set is per-connection), - * so a broad evaluation multiplexes its zone sessions over at most - * `tectonix-worldtree-max-connections` connections instead of opening one per zone — - * keeping the live connection count far below the daemon's per-listener cap. The pool - * is per-`EvalState`, and worldtreed binds one scoped listener per workspace (= per - * Aquifer sandbox), each with its own independent per-listener cap; the Tartarus planner - * runs a single eval per sandbox, so a listener sees one eval's bounded connection set - * rather than the sum across concurrent evals, and the cap is never a host-wide budget - * shared with other sandboxes on the same daemon (the real host-global limit is the - * daemon's fd rlimit). Grown lazily and only when the socket is set; `worldtreePoolNext_` - * is the round-robin cursor. All three are guarded by `worldtreePoolMutex_`. - */ - mutable std::mutex worldtreePoolMutex_; - mutable std::vector> worldtreePool_; - mutable size_t worldtreePoolNext_ = 0; - - /** - * Grants the accessor-side connection-pool unit test (`WorldtreePoolTest` in - * `libexpr-tests/worldtree-pool.cc`) access to the private `acquireWorldtreeZoneSession` - * seam, so it can assert bounded lazy growth, round-robin reuse, and close-on-drop - * against a fake worldtree daemon. Test-only; no production coupling. - */ - friend class WorldtreePoolTest; - /** * Mount a zone by tree SHA, returning a (potentially virtual) store path. * Caches by tree SHA for deduplication across world revisions. @@ -607,39 +578,23 @@ private: StorePath mountZoneByTreeSha(const Hash & treeSha, std::string_view zonePath); /** - * The worldtree control connection. Non-null when `tectonix-worldtree-socket` is set - * and the daemon is reachable; null *only* when the socket is unset (the sole signal - * that callers may use the libgit2 path). When the socket is set but the daemon is - * unreachable or refuses, this THROWS — it never returns null-on-failure and there is - * no libgit2 fallback (fail-loud). Connects at most once. + * The mutable root-checkout control connection. Historical evaluations use the + * immutable FUSE projection and deliberately return null here without falling back + * to libgit2. A configured socket that is needed for a mutable checkout still fails + * loud when unreachable. */ std::shared_ptr worldtreeControlConn() const; /** - * Open a fresh connection to the configured worldtree socket + workspace. Returns null - * *only* when the socket is unset; with the socket set, a connection failure PROPAGATES - * (throws `ProtocolError`) — no warning, no null-on-failure, no fallback (fail-loud; - * see the `worldtreeControlConn()` block comment and the `tectonix-worldtree-socket` - * setting doc). Used for the shared control connection and to populate the bounded - * per-zone session pool (`acquireWorldtreeZoneSession`). + * Open a fresh connection to the configured worldtree socket + workspace for the + * mutable root checkout. Returns null when the socket is unset; a configured socket + * failure propagates. */ std::shared_ptr connectWorldtree() const; /** - * Acquire an ephemeral RO session for `worldPath` pinned at `sha` and scoped to that - * zone's cone, hosted on a connection drawn round-robin from the bounded worldtree - * connection pool (`tectonix-worldtree-max-connections`). The pool grows lazily up to - * the cap, then multiplexes further sessions over the existing connections. The returned - * handle owns the session and releases it (`close_ro` on the hosting connection) when it - * drops. Fail-loud: an unreachable daemon throws (there is no libgit2 fallback). Only - * reached when `tectonix-worldtree-socket` is set (Mode B). - */ - std::shared_ptr - acquireWorldtreeZoneSession(const Hash & sha, const std::string & worldPath) const; - - /** - * Devirtualization tail shared by both worldtree source paths (own-workspace mount - * FS and socket-served foreign-SHA): copy `accessor` to the store (eager) or mount + * Devirtualization tail shared by both worldtree source paths (own-workspace root + * and immutable historical FUSE view): copy `accessor` to the store (eager) or mount * it at a virtual store path (lazy-trees), deduplicating by the zone's `treeSha` * (the daemon's working-tree oid — committed when clean, synthesized when dirty). * The caller picks `accessor`; this owns the store-path identity and cache. diff --git a/src/libutil/include/nix/util/worldtree-client.hh b/src/libutil/include/nix/util/worldtree-client.hh index 5e865fd8d..89b0fa1c7 100644 --- a/src/libutil/include/nix/util/worldtree-client.hh +++ b/src/libutil/include/nix/util/worldtree-client.hh @@ -1,10 +1,9 @@ #pragma once ///@file -/// A self-contained client for the worldtree daemon's control socket (worldtree design -/// §5.1/§5.1a). It speaks the daemon's length-delimited protobuf `Frame` envelope over -/// `AF_UNIX` and exposes the four **Tecnix read verbs** — `dirty_zones`, -/// `zone_tree_shas`, `read_tree`, `read_blobs` — that replace Tectonix's git-binary / -/// libgit2 coupling with O(changes) daemon RPCs. +/// A self-contained client for the mutable worldtree checkout's control socket. It speaks +/// the daemon's length-delimited protobuf `Frame` envelope over `AF_UNIX` and exposes the +/// two O(changes) metadata queries Tecnix needs: `dirty_zones` and `zone_tree_shas`. +/// Committed source bytes are ordinary reads through the worldtree FUSE projection. /// /// It deliberately depends only on the C++ standard library and POSIX sockets — **no /// Nix or libgit2 headers** — for two reasons. First, it must keep working when the @@ -17,7 +16,6 @@ #include #include -#include #include #include #include @@ -83,27 +81,6 @@ struct ZoneSha std::optional treeSha; }; -/// One `read_tree` node of a prefetched subtree skeleton: a path relative to the -/// requested root, its git `mode` (the object type is mode-derived), its `oid`, and — -/// for a blob or symlink — its `size` (0 for a directory), all from object headers -/// with no content fetch. -struct TreeEntry -{ - std::string path; - uint32_t mode; - Oid oid; - uint64_t size; -}; - -/// One `read_blobs` result. `content` is `nullopt` when the object was absent (or -/// present but not a blob — the daemon reports `present=false` rather than failing the -/// batch). -struct Blob -{ - Oid oid; - std::optional content; -}; - /// One `dirty_zones` detail entry: a dirty zone's World path plus its changed working-copy /// files (staged ∪ unstaged), each **relative to the workspace root** (the daemon's /// `StatusEntry.path` form). The daemon computes these while mapping files to zones, so they @@ -115,19 +92,6 @@ struct ZoneDirty std::vector files; }; -/// The result of `scoped.open_ro`: an **ephemeral read-only session** pinned at a base -/// commit (worldtree design §5.1a/§6.1). `ws` *is* the session id — every subsequent read -/// verb ([`Client::readTree`], [`Client::readBlobs`], [`Client::zoneTreeShas`], -/// [`Client::dirtyZones`]) addresses it, so no verb grows a `session_id`. `manifest` is the -/// raw `.meta/manifest.json` bytes at the pinned SHA, delivered in-band (no libgit2, no path -/// traversal). `basePin` is the generation the session refcounts (observability only). -struct RoSession -{ - uint64_t ws; - std::string manifest; - uint64_t basePin; -}; - /// A synchronous, one-call-at-a-time client over a single daemon connection. Not /// thread-safe: a connection multiplexes by request id, but this client issues one /// request and drains to its reply before the next, which is all Tectonix's @@ -159,31 +123,6 @@ public: /// zone yields its committed oid, a dirty zone the synthesized frontier oid. std::vector zoneTreeShas(uint64_t ws, const std::vector & zones); - /// `tecnix.read_tree` — the committed subtree skeleton at workspace-relative `path` - /// (empty ⇒ the root tree), descending `depth` directory levels below the immediate - /// children. Throws [`RpcError`] with `NotFound` for an absent or hidden path. - std::vector readTree(uint64_t ws, const std::string & path, uint32_t depth); - - /// `tecnix.read_blobs` — the decoded bytes of each blob oid, in request order. - std::vector readBlobs(uint64_t ws, const std::vector & oids); - - /// `scoped.open_ro` — open an **ephemeral read-only session** pinned at `baseSha` - /// (20 raw commit-oid bytes), optionally confined to a zone cone. `visibilityMode` is - /// `""`/`"full"` for full visibility (the zone lists must then be empty) or `"scoped"` - /// with `visibleZoneIds`/`visibleZonePaths` naming the cone. Returns the session `ws` - /// + the in-band manifest + the pinned generation. Throws [`RpcError`] with - /// `BaseCommitUnreachable` (retryable) if the SHA is not covered by the current - /// generation, or `Denied` if the connection may not open sessions. - RoSession openRo( - const Oid & baseSha, - const std::string & visibilityMode, - const std::vector & visibleZoneIds, - const std::vector & visibleZonePaths); - - /// `scoped.close_ro` — release a session opened on **this** connection. Idempotent and - /// ownership-gated: closing an id this connection never owned is a clean no-op. - void closeRo(uint64_t ws); - private: explicit Client(int fd); @@ -208,12 +147,6 @@ private: /// an ERROR reply / transport failure). For verbs the daemon does not stream. std::string call(const std::string & method, const std::string & payload); - /// A **streaming** call: issue `method`, then invoke `onChunk` for each RESPONSE chunk - /// in order, returning once STREAM_END arrives (throwing on an ERROR reply). Used by - /// the read verbs, which the daemon serves as a chunked stream (design §6.6). - void callStream( - const std::string & method, const std::string & payload, const std::function & onChunk); - void writeAll(const std::string & bytes); // Buffered frame reader (handles the server's initial CREDIT frame and any diff --git a/src/libutil/worldtree-client.cc b/src/libutil/worldtree-client.cc index ef165a544..15ac0bb23 100644 --- a/src/libutil/worldtree-client.cc +++ b/src/libutil/worldtree-client.cc @@ -63,7 +63,6 @@ constexpr uint64_t KIND_ERROR = 3; // keeps the documentation without complaint. [[maybe_unused]] constexpr uint64_t KIND_CANCEL = 4; [[maybe_unused]] constexpr uint64_t KIND_CREDIT = 5; -constexpr uint64_t KIND_STREAM_END = 6; // The transport's hard per-frame ceiling (the daemon's `MAX_FRAME`, // `wt-controlplane/src/rpc/codec.rs`): 256 MiB. A length prefix larger than this is a @@ -455,32 +454,7 @@ std::string Client::call(const std::string & method, const std::string & payload throw ProtocolError("worldtree: unexpected frame kind in reply to a request"); } -void Client::callStream( - const std::string & method, const std::string & payload, const std::function & onChunk) -{ - uint64_t id = send(method, payload); - // A streamed reply is zero or more RESPONSE chunks (in order) terminated by a single - // STREAM_END, or an ERROR that ends the stream. Unlike a one-shot RESPONSE, the reader - // must consume every frame through the terminator so the connection is left clean for - // the next call. - for (;;) { - RecvFrame fr = recvFor(id); - if (fr.kind == KIND_RESPONSE) { - if (fr.havePayload) - onChunk(fr.payload); - continue; - } - if (fr.kind == KIND_STREAM_END) - return; - if (fr.kind == KIND_ERROR) - throw RpcError( - static_cast(fr.errorCode), - fr.errorMsg.empty() ? "worldtree: daemon returned an error" : fr.errorMsg); - throw ProtocolError("worldtree: unexpected frame kind in a streamed reply"); - } -} - -// ---- the four Tecnix read verbs --------------------------------------------- +// ---- mutable-checkout metadata verbs ---------------------------------------- std::vector Client::dirtyZones(uint64_t ws) { @@ -573,183 +547,4 @@ std::vector Client::zoneTreeShas(uint64_t ws, const std::vector Client::readTree(uint64_t ws, const std::string & path, uint32_t depth) -{ - std::string req; - putVarintField(req, 1, ws); // ReadTreeReq { ws = 1, path = 2 (bytes), depth = 3 } - putLenField(req, 2, path); - putVarintField(req, 3, depth); - - // The daemon streams the skeleton as one or more `ReadTreeResp` chunks (each a subset of - // the entries) terminated by STREAM_END; a small tree arrives as a single chunk. Every - // chunk parses identically — accumulate all of their entries into one vector. - std::vector out; - callStream("tecnix.read_tree", req, [&out](std::string_view chunk) { - Reader r(chunk); - while (!r.atEnd()) { - uint64_t tag = r.varint(); - uint32_t field = static_cast(tag >> 3); - uint32_t wire = static_cast(tag & 0x7); - if (field == 1 && wire == WIRE_LEN) { // ReadTreeResp { entries = 1 (repeated TreeEntry) } - Reader e(r.bytes()); - TreeEntry entry{}; - std::optional oid; - while (!e.atEnd()) { - uint64_t etag = e.varint(); - uint32_t efield = static_cast(etag >> 3); - uint32_t ewire = static_cast(etag & 0x7); - switch (efield) { - case 1: // path (bytes) - if (ewire != WIRE_LEN) - throw ProtocolError("worldtree: bad wire type for tree-entry path"); - entry.path = std::string(e.bytes()); - break; - case 2: // mode (uint32) - if (ewire != WIRE_VARINT) - throw ProtocolError("worldtree: bad wire type for tree-entry mode"); - entry.mode = static_cast(e.varint()); - break; - case 3: // oid (bytes) - if (ewire != WIRE_LEN) - throw ProtocolError("worldtree: bad wire type for tree-entry oid"); - oid = toOid(e.bytes()); - break; - case 4: // size (uint64) - if (ewire != WIRE_VARINT) - throw ProtocolError("worldtree: bad wire type for tree-entry size"); - entry.size = e.varint(); - break; - default: - e.skip(ewire); - break; - } - } - // A tree node always carries a 20-byte oid; an empty/absent one is malformed. - if (!oid) - throw ProtocolError("worldtree: tree entry missing its object id"); - entry.oid = *oid; - out.push_back(std::move(entry)); - } else - r.skip(wire); - } - }); - return out; -} - -std::vector Client::readBlobs(uint64_t ws, const std::vector & oids) -{ - std::string req; - putVarintField(req, 1, ws); // ReadBlobsReq { ws = 1, oids = 2 (repeated bytes) } - for (const auto & oid : oids) - putLenFieldAlways(req, 2, std::string_view(reinterpret_cast(oid.data()), oid.size())); - - // Streamed as one or more `ReadBlobsResp` chunks (each a subset of the requested blobs) - // terminated by STREAM_END. Accumulate every chunk's blobs, preserving request order - // (the daemon emits them in the order asked). - std::vector out; - callStream("tecnix.read_blobs", req, [&out](std::string_view chunk) { - Reader r(chunk); - while (!r.atEnd()) { - uint64_t tag = r.varint(); - uint32_t field = static_cast(tag >> 3); - uint32_t wire = static_cast(tag & 0x7); - if (field == 1 && wire == WIRE_LEN) { // ReadBlobsResp { blobs = 1 (repeated Blob) } - Reader e(r.bytes()); - std::optional oid; - std::string content; - bool present = false; - while (!e.atEnd()) { - uint64_t etag = e.varint(); - uint32_t efield = static_cast(etag >> 3); - uint32_t ewire = static_cast(etag & 0x7); - switch (efield) { - case 1: // oid (bytes) - if (ewire != WIRE_LEN) - throw ProtocolError("worldtree: bad wire type for blob oid"); - oid = toOid(e.bytes()); - break; - case 2: // content (bytes) - if (ewire != WIRE_LEN) - throw ProtocolError("worldtree: bad wire type for blob content"); - content = std::string(e.bytes()); - break; - case 3: // present (bool) - if (ewire != WIRE_VARINT) - throw ProtocolError("worldtree: bad wire type for blob present"); - present = e.varint() != 0; - break; - default: - e.skip(ewire); - break; - } - } - Blob blob; - blob.oid = oid.value_or(Oid{}); - if (present) - blob.content = std::move(content); - out.push_back(std::move(blob)); - } else - r.skip(wire); - } - }); - return out; -} - -// ---- session lifecycle: scoped.open_ro / scoped.close_ro -------------------- - -RoSession Client::openRo( - const Oid & baseSha, - const std::string & visibilityMode, - const std::vector & visibleZoneIds, - const std::vector & visibleZonePaths) -{ - std::string req; - // OpenRoReq { base_sha = 1 (bytes), visibility_mode = 2 (string), - // visible_zone_ids = 3 (repeated string), visible_zone_paths = 4 (repeated string) } - putLenFieldAlways(req, 1, std::string_view(reinterpret_cast(baseSha.data()), baseSha.size())); - putLenField(req, 2, visibilityMode); - for (const auto & z : visibleZoneIds) - putLenFieldAlways(req, 3, z); - for (const auto & z : visibleZonePaths) - putLenFieldAlways(req, 4, z); - - std::string resp = call("scoped.open_ro", req); - - RoSession out{}; - Reader r(resp); - while (!r.atEnd()) { - uint64_t tag = r.varint(); - uint32_t field = static_cast(tag >> 3); - uint32_t wire = static_cast(tag & 0x7); - switch (field) { - case 1: // ws (uint64) - if (wire != WIRE_VARINT) - throw ProtocolError("worldtree: bad wire type for open_ro ws"); - out.ws = r.varint(); - break; - case 2: // manifest (bytes) - if (wire != WIRE_LEN) - throw ProtocolError("worldtree: bad wire type for open_ro manifest"); - out.manifest = std::string(r.bytes()); - break; - case 3: // base_pin (uint64) - if (wire != WIRE_VARINT) - throw ProtocolError("worldtree: bad wire type for open_ro base_pin"); - out.basePin = r.varint(); - break; - default: - r.skip(wire); - break; - } - } - return out; -} - -void Client::closeRo(uint64_t ws) -{ - std::string req; - putVarintField(req, 1, ws); // CloseRoReq { ws = 1 } - (void) call("scoped.close_ro", req); // CloseRoResp is empty; idempotent, ownership-gated. -} - } // namespace nix::worldtree diff --git a/src/libutil/worldtree-smoke.cc b/src/libutil/worldtree-smoke.cc deleted file mode 100644 index 8ffcc51cc..000000000 --- a/src/libutil/worldtree-smoke.cc +++ /dev/null @@ -1,174 +0,0 @@ -///@file -/// A standalone exerciser for the worldtree C++ client (`worldtree-client.hh`), used by -/// the cross-language wire-compatibility smoke. It connects to a running worldtreed, -/// drives all four Tecnix read verbs against a workspace the harness has already set up, -/// and prints each result as a parseable line. It closes with a fail-loud probe: a read of -/// an unknown workspace must surface the daemon's ERROR as a thrown exception, never a silent -/// empty success. The harness (a worldtreed Rust example) -/// owns the fixture and asserts on this output, so this program stays a dumb, faithful -/// exerciser — its only job is to prove the C++ codec and the daemon's prost encoder -/// agree on the wire in *both* directions, against real prost (not a golden fixture). -/// -/// It is compiled directly with `clang++` against `worldtree-client.cc` (no Nix build, -/// no protobuf-c) — the Tecnix Nix build is, at the time of writing, blocked on an -/// unrelated missing-libgit2 dependency, and this seam must be testable regardless. -/// -/// Usage: worldtree-smoke --socket PATH --ws N - -#include "nix/util/worldtree-client.hh" - -#include -#include -#include -#include - -using namespace nix::worldtree; - -namespace { - -std::string toHex(const Oid & oid) -{ - static const char * digits = "0123456789abcdef"; - std::string s; - s.reserve(40); - for (uint8_t b : oid) { - s.push_back(digits[b >> 4]); - s.push_back(digits[b & 0xf]); - } - return s; -} - -/// Render bytes printably: as text if all bytes are printable ASCII/whitespace, else as -/// hex. The harness sets blob content to plain text, so the text form is what it greps. -std::string render(const std::string & bytes) -{ - bool printable = true; - for (unsigned char c : bytes) - if (!(c == '\n' || c == '\t' || (c >= 0x20 && c < 0x7f))) { - printable = false; - break; - } - if (printable) { - // One-line: escape newlines so a multi-line blob stays a single output line. - std::string out; - for (char c : bytes) - out += (c == '\n') ? std::string("\\n") : std::string(1, c); - return "text:" + out; - } - std::string out = "hex:"; - static const char * digits = "0123456789abcdef"; - for (unsigned char c : bytes) { - out.push_back(digits[c >> 4]); - out.push_back(digits[c & 0xf]); - } - return out; -} - -[[noreturn]] void usage() -{ - std::fprintf(stderr, "usage: worldtree-smoke --socket PATH --ws N\n"); - std::exit(2); -} - -} // namespace - -int main(int argc, char ** argv) -{ - std::string socketPath; - uint64_t ws = 0; - bool haveWs = false; - - for (int i = 1; i < argc; i++) { - std::string arg = argv[i]; - auto next = [&]() -> std::string { - if (i + 1 >= argc) - usage(); - return argv[++i]; - }; - if (arg == "--socket") - socketPath = next(); - else if (arg == "--ws") { - ws = std::strtoull(next().c_str(), nullptr, 10); - haveWs = true; - } else - usage(); - } - if (socketPath.empty() || !haveWs) - usage(); - - try { - Client client = Client::connect(socketPath); - - // 1. dirty_zones — which zones have tracked working-copy changes. - auto dirty = client.dirtyZones(ws); - for (const auto & z : dirty) - std::printf("DIRTY %s\n", z.c_str()); - - // 2a. zone_tree_shas with an empty request — expands to the dirty set, the common - // Tecnix call. Each dirty zone yields its synthesized working-tree oid. - auto zts = client.zoneTreeShas(ws, {}); - for (const auto & e : zts) - std::printf("ZTS %s %s\n", e.zone.c_str(), e.treeSha ? toHex(*e.treeSha).c_str() : "absent"); - - // 2b. zone_tree_shas with an explicit list mixing a real (dirty) zone and one - // that does not exist. This proves the response stays one-to-one with the - // request and that the daemon's empty-bytes "absent" marker decodes to a - // `nullopt` treeSha on the C++ side (not an empty-but-present oid). - if (!dirty.empty()) { - std::vector probe = {dirty.front(), "//does/not/exist"}; - auto explicitZts = client.zoneTreeShas(ws, probe); - for (const auto & e : explicitZts) - std::printf("ZTS2 %s %s\n", e.zone.c_str(), e.treeSha ? toHex(*e.treeSha).c_str() : "absent"); - } - - // 3. read_tree at the root, deep — the whole committed skeleton (overlay-free). - // Collect every blob oid we see for the read_blobs round-trip below. - auto tree = client.readTree(ws, "", 64); - std::vector blobOids; - for (const auto & e : tree) { - std::printf( - "TREE %s mode=%o oid=%s size=%llu\n", - e.path.c_str(), - e.mode, - toHex(e.oid).c_str(), - static_cast(e.size)); - // A regular/executable file (not a tree, not a symlink): mode 0o100xxx. - if ((e.mode & 0170000) == 0100000) - blobOids.push_back(e.oid); - } - - // 4. read_blobs — fetch each committed blob by oid and render its bytes. This - // closes the loop: the harness staged known content, the daemon hashed and - // stored it, and we read it back by the oid read_tree reported. - if (!blobOids.empty()) { - auto blobs = client.readBlobs(ws, blobOids); - for (const auto & b : blobs) - std::printf("BLOB %s %s\n", toHex(b.oid).c_str(), b.content ? render(*b.content).c_str() : "absent"); - } - - // 5. fail-loud: a read of an unknown workspace must surface the daemon's ERROR as a - // thrown RpcError, never a silent empty success. Probe with a ws the harness never - // provisioned (the bitwise complement of the real one) and require the throw; a - // returned result would be a wire/decoding bug the harness must catch. - try { - const uint64_t unknownWs = ~ws; - client.readTree(unknownWs, "", 0); - std::fprintf( - stderr, - "worldtree-smoke: read of unknown ws %llu returned instead of throwing\n", - static_cast(unknownWs)); - return 1; - } catch (const RpcError &) { - std::printf("FAILLOUD ok\n"); - } - - std::printf("SMOKE-DONE\n"); - return 0; - } catch (const RpcError & e) { - std::fprintf(stderr, "worldtree-smoke: daemon error (code %d): %s\n", static_cast(e.code), e.what()); - return 1; - } catch (const std::exception & e) { - std::fprintf(stderr, "worldtree-smoke: %s\n", e.what()); - return 1; - } -}