Make the embed endpoint optional so lexical-only search works

.env.example

@@ -6,9 +6,13 @@
 DATABASE_URL=postgres://memory:memory-dev-password@localhost:5434/memory
 DB_POOL_MAX=8
 
-# Embeddings (compose.yml → Ollama on :11434). The engine never embeds
-# in-process; it always calls this endpoint. Swap four vars for a hosted
-# provider (e.g. EMBED_BASE_URL=https://api.openai.com, EMBED_API_STYLE=openai).
+# Embeddings (compose.yml → Ollama on :11434). Optional — leave both
+# EMBED_BASE_URL and EMBED_MODEL unset to run lexical-only (no dense
+# retrieval; `add` and lexical `search` still work,
+# degraded: ["dense_unavailable", "lexical_only"]). When set, both are
+# required together. The engine never embeds in-process; it always calls
+# this endpoint. Swap four vars for a hosted provider (e.g.
+# EMBED_BASE_URL=https://api.openai.com, EMBED_API_STYLE=openai).
 EMBED_BASE_URL=http://localhost:11434
 EMBED_MODEL=nomic-embed-text
 EMBED_API_STYLE=ollama

CHANGELOG.md

@@ -25,6 +25,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- `loadMemoryConfig` / `EngineConfig.embed` no longer requires an embed
+  endpoint: a host with a pgvector Postgres and no embed endpoint now
+  constructs and serves `add` + lexical `search`. Dense retrieval is skipped
+  (not attempted-and-failed) and `search` reports
+  `degraded: ["dense_unavailable", "lexical_only"]` so the state stays
+  observable (CL-6287). **Migration note:** `dense_unavailable` is now also
+  emitted on every search for a deliberately-unconfigured engine, not only on
+  a transient dense-retrieval failure — a host with an existing alert rule
+  keyed on `dense_unavailable` alone should also check for `lexical_only` in
+  the same `degraded` array to distinguish "opted into lexical-only" from an
+  actual regression.
+- `add`'s `degraded` is now a reason array (`["embed_unavailable"]` and/or
+  `["embed_unavailable", "lexical_only"]`), matching `search`'s shape —
+  previously a bare boolean, which made it impossible to write one
+  "is this response degraded" check across both verbs (CL-6287). **Breaking
+  if a host coded against the boolean:** `degraded: true` is now
+  `degraded: [...]`; check array presence/length instead of truthiness (both
+  are still falsy/omitted when the document captured cleanly).
+- `Memory.capabilities.embeddingsConfigured` (and the underlying
+  `DocumentStore.capabilities`) let a host learn recall is lexical-only at
+  construction time, without issuing a search first (CL-6287).
 - Feed `nextCursor` advances past the examined raw page after grant-tag
   post-filter (a fully denied page no longer stalls the consumer forever).
 - Distiller default system prompt uses the configured `agentId` for

IMPLEMENTATION.md

@@ -88,14 +88,43 @@ fallback)` parses a positive integer or throws.
 |  `DATABASE_URL` | **yes** | — | the engine's own pgvector Postgres |
 | `DB_POOL_MAX` | no | `8` | postgres-js pool size |
 | `FTS_LANGUAGE` | no | `english` | text search config for the lexical channel; fixed into the generated column at migration time — changing it later requires rebuilding the column (recipe below), and `runMemoryMigrations` fails loudly if config and column disagree. Unqualified `pg_catalog` config names only — a schema-qualified config (`myschema.mycfg`) is rejected explicitly, both when configuring and when read back from an already-migrated column. |
-| `EMBED_BASE_URL` | **yes** | — | embed endpoint root, no path suffix |
-| `EMBED_MODEL` | **yes** | — | model id/name passed to the embed endpoint |
+| `EMBED_BASE_URL` | no | — | embed endpoint root, no path suffix; absent (with `EMBED_MODEL` also absent) => lexical-only, see below |
+| `EMBED_MODEL` | no | — | model id/name passed to the embed endpoint; must be set together with `EMBED_BASE_URL` (both or neither — one without the other throws) |
 | `EMBED_API_STYLE` | no | `"openai"` | `"openai" \| "tei" \| "ollama"` |
 | `EMBED_API_KEY` | no | `undefined` | forwarded as `Authorization: Bearer <key>` |
 | `RERANK_BASE_URL` | no | `undefined` | absent => search degrades to fusion-only |
 | `RERANK_MODEL` | no | `undefined` | defaults to `bge-reranker-v2-m3` in the client |
 | `RERANK_API_KEY` | no | `undefined` | forwarded as Bearer token to the rerank endpoint |
 
+**Lexical-only mode (CL-6287).** `EngineConfig.embed` is optional — leave both
+`EMBED_BASE_URL`/`EMBED_MODEL` unset and the engine still constructs and
+serves `add` + lexical `search` against a pgvector Postgres with no
+embed endpoint configured. Dense retrieval is skipped rather than
+attempted (no doomed HTTP call on every query), `add` still captures
+documents (chunks stored, no vectors), and both verbs report a `degraded`
+reason array — never a bare boolean, so a host can write one "is this
+response degraded" check across both: `search` reports
+`degraded: ["dense_unavailable", "lexical_only"]`; `add` reports
+`degraded: ["embed_unavailable", "lexical_only"]` (or `["embed_unavailable"]`
+alone when the endpoint IS configured but a specific embed pass failed — a
+client error, timeout, or rejected chunk). The embed-model registry
+(`ensureEmbedModel`/`activateEmbedModel`) is never reached in this mode.
+
+**Discoverability.** A host does not have to run a search to learn recall is
+limited: `memory.capabilities.embeddingsConfigured` (on the `Memory` handle
+`createMemory` returns) is `false` for a lexical-only engine, `true`
+otherwise — known at construction, no query needed. A custom `documentStore`
+that doesn't report its own `capabilities` defaults to `true` (this SDK
+cannot introspect a vendor store it doesn't own); see
+`DocumentStoreCapabilities` (ports/types.ts) for how a vendor store opts in.
+
+The replay/backfill pipeline (`runTransform`, `promoteGeneration` in
+`services/transform.ts`) still requires an embed endpoint — re-deriving a
+corpus is inherently a re-embedding operation — and fails loudly if run
+against an engine with none configured; re-embedding documents captured while
+lexical-only, once an endpoint is later added, is an open follow-up (not
+implemented).
+
 The engine's `EngineConfig.rerank` carries
 no `apiStyle` field of its own; `search.ts`'s `toRerankClientConfig` hardcodes
 `apiStyle: "tei"` when building the client config, i.e. the engine currently
@@ -688,9 +717,9 @@ bun run test                                           # unit suite (no external
 
 `compose.yml` provisions the pgvector Postgres (`memory` db, host port
 `5434`), an Ollama embeddings server (`:11434`), and a TEI reranker (`:8085`).
-The engine **never embeds internally** — `EMBED_BASE_URL` must point at a real
-endpoint. A model endpoint is just a URL + capability options, trusted the same
-as `DATABASE_URL`:
+The engine **never embeds internally** — when `EMBED_BASE_URL` is set, it must
+point at a real endpoint. A model endpoint is just a URL + capability options,
+trusted the same as `DATABASE_URL`:
 
 - **Local default**: Ollama at `http://localhost:11434`
   (`EMBED_API_STYLE=ollama`, `EMBED_MODEL=nomic-embed-text`).
@@ -701,6 +730,11 @@ as `DATABASE_URL`:
 `RERANK_BASE_URL` is optional — unset runs lexical+dense+MMR without the
 cross-encoder (`degraded: ["rerank_unavailable"]`, still ranked/citable hits).
 
+`EMBED_BASE_URL`/`EMBED_MODEL` are optional too — unset both to run
+lexical-only (`degraded: ["dense_unavailable", "lexical_only"]`, no dense
+channel, `add` still captures documents unvectorized). See the lexical-only
+note above.
+
 ## Testing
 
 `bun test ./src` (`bun run test`), coverage via `bun run test:coverage`. Every

src/config.ts

@@ -21,7 +21,16 @@ export type EngineConfig = {
   // trusted the same as DATABASE_URL — including a self-hosted endpoint on
   // localhost or a private IP. Self-hosted or managed makes no difference:
   // there is no self-host flag anywhere in the engine.
-  embed: {
+  //
+  // Absent entirely => no embed endpoint configured. The engine still
+  // constructs and serves `add` + lexical `search` in that case: dense
+  // retrieval is skipped (never attempted, so it never runs a doomed HTTP
+  // call), capture stores chunks without vectors, and search reports
+  // `["dense_unavailable", "lexical_only"]` so the state is observable
+  // rather than silent (see hybridSearch in services/search.ts). A pgvector
+  // Postgres with no embed endpoint is a legitimate, fully-capable
+  // lexical-only deployment, not a misconfiguration.
+  embed?: {
     baseUrl: string;
     model: string;
     apiStyle: string;

src/core/degrade-metrics.ts

@@ -25,13 +25,23 @@ const DEGRADE_FLAG_SET = {
   live_timeout: true,
   live_error: true,
   memory_unavailable: true,
+  lexical_only: true,
 } satisfies Record<DegradeFlag, true>;
 
 
 // Deriving the list from a `satisfies Record<DegradeFlag, true>` object
 // means adding a flag in hybrid-search.ts without adding it here is a
 // compile error (missing property), not a silent gap that only a test
 // iterating this same constant could ever have caught.
+//
+// `lexical_only` sits at a permanent ~100% windowed rate for a host that has
+// deliberately opted out of dense retrieval (no embed endpoint configured) —
+// unlike every other flag here, that is the intended, steady state rather
+// than a regression, so it escalates to log.error and stays there for as
+// long as the host runs lexical-only. That is accurate (the health snapshot
+// should show "running degraded"), not a bug; a host that finds the
+// permanent log.error noisy can raise its own highWatermark via
+// `configureDegradeMetrics`.
 export const ALL_DEGRADE_FLAGS: readonly DegradeFlag[] = Object.keys(
   DEGRADE_FLAG_SET,
 ) as DegradeFlag[];

src/core/embed-worker.ts

@@ -10,6 +10,19 @@ export interface EmbeddableChunk {
   text: string;
 }
 
+// Capture's counterpart to search's `DegradeFlag` (hybrid-search.ts) — an
+// array, never a bare boolean, so a host can write one "is this response
+// degraded" check across `add` and `search`. Lives here (not services/
+// capture.ts) so `ports/types.ts` can reference it for
+// `DocumentStoreAddResult` the same way it already references `DegradeFlag`
+// for `DocumentStoreSearchResult`. `embed_unavailable` is the embed pass's
+// counterpart to `dense_unavailable` — it ran and failed (client error,
+// timeout, or a rejected/dims-mismatched chunk); `lexical_only` is paired
+// with it specifically when there's no embed endpoint configured at all,
+// mirroring search's `dense_unavailable`/`lexical_only` pairing for the same
+// "configured off" state.
+export type CaptureDegradedReason = "embed_unavailable" | "lexical_only";
+
 export interface EmbedChunksResult {
   embedded: number;
   rejected: Array<{ chunkId: string; reason: string }>;

src/core/engine-client-config.ts

@@ -17,10 +17,14 @@ const VALID_EMBED_API_STYLES = new Set(["openai", "tei", "ollama"]);
 // the trust boundary between config and the client — an invalid value is an
 // operator misconfiguration and must fail loudly, not silently degrade.
 // Built from the engine's own operator-configured embed endpoint — a trusted
-// URL, the same as DATABASE_URL.
+// URL, the same as DATABASE_URL. Absent `embed` => no embed endpoint
+// configured => `undefined`, the same degrade-soft precedent already used by
+// `toRerankClientConfig` below — a caller must skip dense retrieval / the
+// embed pass entirely rather than dispatch a client with no endpoint.
 export function toEmbedClientConfig(
   embed: EngineConfig["embed"],
-): EmbedClientConfig {
+): EmbedClientConfig | undefined {
+  if (!embed) return undefined;
   if (!VALID_EMBED_API_STYLES.has(embed.apiStyle)) {
     throw new Error(
       `Invalid EMBED_API_STYLE "${embed.apiStyle}" — must be one of: ${[...VALID_EMBED_API_STYLES].join(", ")}`,

src/core/hybrid-search.ts

@@ -20,7 +20,16 @@ export type DegradeFlag =
   | "rerank_query_too_long"
   | "live_timeout"
   | "live_error"
-  | "memory_unavailable";
+  | "memory_unavailable"
+  // The engine has no embed endpoint configured at all (EngineConfig.embed
+  // is absent) — a deliberate, structural lexical-only deployment, distinct
+  // from `dense_unavailable`'s per-call "dense contributed nothing this
+  // time" (which also covers a configured endpoint that's merely down, or a
+  // tenant with no active embed model yet). Always paired with
+  // `dense_unavailable` on the search response (see hybridSearch,
+  // services/search.ts) so an aggregate degrade-rate consumer still sees
+  // "dense didn't contribute" even if it only understands that one flag.
+  | "lexical_only";
 
 export interface RankedCandidate {
   /** Stable identifier the candidate is keyed by across channels (a chunk id). */

src/index.ts

@@ -38,6 +38,7 @@ export type {
   HybridSearchResult,
   MemoryAddParams,
   MemoryAddResult,
+  MemoryCapabilities,
   MemorySearchParams,
   MemoryIdentity,
   Memory,
@@ -77,6 +78,7 @@ export type {
   DocumentStore,
   DocumentStoreAddParams,
   DocumentStoreAddResult,
+  DocumentStoreCapabilities,
   DocumentStoreSearchItem,
   DocumentStoreSearchParams,
   DocumentStoreSearchResult,

src/memory.test.ts

@@ -173,6 +173,57 @@ describe("createMemory — construction validation", () => {
   });
 });
 
+// CL-6287 review: a consumer (settings page, health check) must be able to
+// learn recall is lexical-only WITHOUT issuing a search first.
+describe("createMemory — capabilities.embeddingsConfigured (CL-6287)", () => {
+  it("reports true when EngineConfig.embed is configured", async () => {
+    const plane = createMemory({
+      config: baseConfig({
+        baseUrl: undefined,
+        model: undefined,
+        apiKey: undefined,
+        maxDocChars: undefined,
+        timeoutMs: undefined,
+      }),
+    });
+    expect(plane.capabilities.embeddingsConfigured).toBe(true);
+    await plane.close();
+  });
+
+  it("reports false when EngineConfig.embed is absent (lexical-only)", async () => {
+    const config: MemoryConfig = {
+      memory: {
+        databaseUrl: "postgres://localhost:5432/nonexistent-test-db",
+        dbPoolMax: 1,
+        ftsLanguage: "english",
+        rerank: {
+          baseUrl: undefined,
+          model: undefined,
+          apiKey: undefined,
+          maxDocChars: undefined,
+          timeoutMs: undefined,
+        },
+      },
+    };
+    const plane = createMemory({ config });
+    expect(plane.capabilities.embeddingsConfigured).toBe(false);
+    await plane.close();
+  });
+
+  it("defaults to true for a custom DocumentStore that doesn't report its own capabilities", async () => {
+    const plane = createMemory({
+      documentStore: {
+        add: async () => ({ documentId: "d1", versionId: "v1" }),
+        search: async () => ({ items: [] }),
+        list: async () => [],
+        close: async () => {},
+      },
+    });
+    expect(plane.capabilities.embeddingsConfigured).toBe(true);
+    await plane.close();
+  });
+});
+
 describe("createMemory.find — grant-tag post-filter wiring", () => {
   const hybridSearch = mock((): Promise<HybridSearchResult> =>
     Promise.resolve({

src/memory.ts

@@ -14,7 +14,7 @@ import { createFtsVerification, parseFtsLanguage } from "./core/fts-language.ts"
 import { createRawSqlClient } from "./core/embed-sql.ts";
 import type { SearchHit } from "./core/schemas/search.ts";
 import { validateRerankConfig } from "./core/rerank-client.ts";
-import { captureDocument } from "./services/capture.ts";
+import { captureDocument, type CaptureDegradedReason } from "./services/capture.ts";
 import {
   hybridSearch,
   MemorySearchInputError,
@@ -70,6 +70,7 @@ import type { MemoryConfig } from "./mount-config.ts";
 import type { GrantConfig } from "./routes/deps.ts";
 import type {
   DocumentStore,
+  DocumentStoreCapabilities,
   DocumentStoreSearchParams,
   SourceProvider,
 } from "./ports/types.ts";
@@ -89,9 +90,13 @@ export type { SearchHit } from "./core/schemas/search.ts";
 export type {
   DocumentStore,
   DocumentStoreAddParams,
+  DocumentStoreCapabilities,
   LiveSearchItem,
   SourceProvider,
 } from "./ports/types.ts";
+// Alias so hosts read `MemoryCapabilities` (the name on the `Memory` handle
+// they actually hold) rather than reaching for the port-level type name.
+export type MemoryCapabilities = DocumentStoreCapabilities;
 export {
   SEARCH_LIMIT_MIN,
   SEARCH_LIMIT_MAX,
@@ -200,6 +205,13 @@ export type MemoryAddResult = {
    * store or soft failure). Omitted when no peer share was requested.
    */
   grantsMaterialized?: boolean;
+  /**
+   * Mirrors `search`'s `degraded` (a reason array, never a bare boolean) so
+   * a host can write one "is this response degraded" check across both
+   * verbs. Omitted when the document captured cleanly. See
+   * `CaptureDegradedReason` (services/capture.ts).
+   */
+  degraded?: CaptureDegradedReason[];
 };
 
 export type SearchAttribution = {
@@ -283,6 +295,14 @@ export type Memory = {
   search(params: MemorySearchParams): Promise<SearchResult>;
   add(params: MemoryAddParams): Promise<MemoryAddResult>;
   list(params: MemoryListParams): Promise<TimelineEvent[]>;
+  /**
+   * Static capability facts, known at construction — check
+   * `embeddingsConfigured` to learn recall is lexical-only WITHOUT issuing
+   * a search first (CL-6287). Always present; a custom DocumentStore that
+   * doesn't report its own capabilities defaults to
+   * `embeddingsConfigured: true` (see DocumentStoreCapabilities).
+   */
+  readonly capabilities: MemoryCapabilities;
   /**
    * Cursor pull of new live versions (engine store only). Grant-checked like
    * search. See docs/FEED.md.
@@ -788,7 +808,17 @@ function createPlaneFromStore(
     });
   }
 
+  // A custom store that doesn't report its own capabilities is assumed
+  // embeddings-capable — the pre-CL-6287 default, since this SDK cannot
+  // introspect a vendor store it doesn't own. The engine store always
+  // reports one (see createEngineDocumentStore).
+  const capabilities: MemoryCapabilities = store.capabilities ?? {
+    embeddingsConfigured: true,
+  };
+
   const plane: Memory = {
+    capabilities,
+
     async search(params) {
       return searchMerged(params);
     },
@@ -1294,6 +1324,9 @@ function createEngineDocumentStore(config: MemoryConfig): {
         return {
           documentId: captureResult.documentId,
           versionId: captureResult.versionId,
+          ...(captureResult.status === "captured" && captureResult.degraded
+            ? { degraded: captureResult.degraded }
+            : {}),
         };
       },
 
@@ -1415,6 +1448,8 @@ function createEngineDocumentStore(config: MemoryConfig): {
       async close() {
         await sql.end({ timeout: 5 });
       },
+
+      capabilities: { embeddingsConfigured: Boolean(engineConfig.embed) },
     },
     deps,
   };