feat(execution): allow custom tool discovery providers

packages/core/execution/README.md

@@ -51,6 +51,25 @@ console.log(result);
 // { result: 12, logs: [...] }
 ```
 
+## Custom tool discovery
+
+`tools.search(...)` uses Executor's built-in lexical tool discovery by default. Hosts can provide their own implementation, such as an indexed or semantic search provider, without replacing the sandbox runtime:
+
+```ts
+import { createExecutionEngine, type ToolDiscoveryProvider } from "@executor-js/execution";
+
+const toolDiscoveryProvider: ToolDiscoveryProvider = {
+  searchTools: ({ query, namespace, limit, offset }) =>
+    mySearchIndex.searchTools({ query, namespace, limit, offset }),
+};
+
+const engine = createExecutionEngine({
+  executor,
+  codeExecutor: makeQuickJsExecutor(),
+  toolDiscoveryProvider,
+});
+```
+
 ## Pause/resume for elicitation
 
 When the host doesn't support inline elicitation, use `executeWithPause` to intercept the first request as a pause point:

packages/core/execution/src/engine.ts

@@ -12,10 +12,11 @@ import { CodeExecutionError } from "@executor-js/codemode-core";
 import type { CodeExecutor, ExecuteResult, SandboxToolInvoker } from "@executor-js/codemode-core";
 
 import {
+  defaultToolDiscoveryProvider,
   makeExecutorToolInvoker,
-  searchTools,
   listExecutorSources,
   describeTool,
+  type ToolDiscoveryProvider,
 } from "./tool-invoker";
 import { ExecutionToolError } from "./errors";
 import { buildExecuteDescription } from "./description";
@@ -27,6 +28,7 @@ import { buildExecuteDescription } from "./description";
 export type ExecutionEngineConfig<E extends Cause.YieldableError = CodeExecutionError> = {
   readonly executor: Executor;
   readonly codeExecutor: CodeExecutor<E>;
+  readonly toolDiscoveryProvider?: ToolDiscoveryProvider;
 };
 
 export type ExecutionResult =
@@ -186,7 +188,11 @@ const readOptionalOffset = (value: unknown, toolName: string): number | Executio
   return Math.floor(value);
 };
 
-const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): SandboxToolInvoker => {
+const makeFullInvoker = (
+  executor: Executor,
+  invokeOptions: InvokeOptions,
+  toolDiscoveryProvider: ToolDiscoveryProvider,
+): SandboxToolInvoker => {
   const base = makeExecutorToolInvoker(executor, { invokeOptions });
   return {
     invoke: ({ path, args }) => {
@@ -226,14 +232,19 @@ const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): Sand
           return Effect.fail(offset);
         }
 
-        return searchTools(executor, args.query ?? "", limit, {
-          namespace: args.namespace,
-          offset,
-        }).pipe(
-          Effect.withSpan("mcp.tool.dispatch", {
-            attributes: { "mcp.tool.name": path, "executor.tool.builtin": true },
-          }),
-        );
+        return toolDiscoveryProvider
+          .searchTools({
+            executor,
+            query: args.query ?? "",
+            limit,
+            namespace: args.namespace,
+            offset,
+          })
+          .pipe(
+            Effect.withSpan("mcp.tool.dispatch", {
+              attributes: { "mcp.tool.name": path, "executor.tool.builtin": true },
+            }),
+          );
       }
       if (path === "executor.sources.list") {
         if (args !== undefined && !isRecord(args)) {
@@ -364,7 +375,7 @@ export type ExecutionEngine<E extends Cause.YieldableError = CodeExecutionError>
 export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecutionError>(
   config: ExecutionEngineConfig<E>,
 ): ExecutionEngine<E> => {
-  const { executor, codeExecutor } = config;
+  const { executor, codeExecutor, toolDiscoveryProvider = defaultToolDiscoveryProvider } = config;
   const pausedExecutions = new Map<string, InternalPausedExecution<E>>();
   let nextId = 0;
 
@@ -433,7 +444,11 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
         return yield* Deferred.await(responseDeferred);
       });
 
-    const invoker = makeFullInvoker(executor, { onElicitation: elicitationHandler });
+    const invoker = makeFullInvoker(
+      executor,
+      { onElicitation: elicitationHandler },
+      toolDiscoveryProvider,
+    );
     fiber = yield* Effect.forkDetach(
       codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec")),
     );
@@ -477,9 +492,13 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu
       "mcp.execute.mode": "inline",
       "mcp.execute.code_length": code.length,
     });
-    const invoker = makeFullInvoker(executor, {
-      onElicitation: options.onElicitation,
-    });
+    const invoker = makeFullInvoker(
+      executor,
+      {
+        onElicitation: options.onElicitation,
+      },
+      toolDiscoveryProvider,
+    );
     return yield* codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec"));
   });
 

packages/core/execution/src/index.ts

@@ -12,8 +12,13 @@ export {
 export { buildExecuteDescription } from "./description";
 export { ExecutionToolError } from "./errors";
 export {
+  defaultToolDiscoveryProvider,
   makeExecutorToolInvoker,
   searchTools,
   listExecutorSources,
   describeTool,
+  type ToolDiscoveryInput,
+  type ToolDiscoveryProvider,
+  type PagedResult,
+  type ToolDiscoveryResult,
 } from "./tool-invoker";

packages/core/execution/src/tool-invoker.test.ts

@@ -12,7 +12,12 @@ import {
 import { makeTestConfig, typeCheckOutputTypeScript } from "@executor-js/sdk/testing";
 import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs";
 import { createExecutionEngine } from "./engine";
-import { describeTool, makeExecutorToolInvoker, searchTools } from "./tool-invoker";
+import {
+  describeTool,
+  makeExecutorToolInvoker,
+  searchTools,
+  type ToolDiscoveryProvider,
+} from "./tool-invoker";
 
 const codeExecutor = makeQuickJsExecutor();
 
@@ -334,6 +339,79 @@ describe("tool discovery", () => {
     }),
   );
 
+  it.effect("lets execution hosts provide custom tool discovery", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+      const calls: Array<{
+        readonly query: string;
+        readonly namespace?: string;
+        readonly limit: number;
+        readonly offset: number;
+      }> = [];
+      const provider: ToolDiscoveryProvider = {
+        searchTools: ({ query, namespace, limit, offset }) =>
+          Effect.sync(() => {
+            calls.push({ query, namespace, limit, offset });
+            return {
+              items: [
+                {
+                  path: "custom.searchResult",
+                  name: "searchResult",
+                  description: "Provided by the host",
+                  sourceId: "custom",
+                  score: 999,
+                },
+              ],
+              total: 1,
+              hasMore: false,
+              nextOffset: null,
+            };
+          }),
+      };
+      const engine = createExecutionEngine({
+        executor,
+        codeExecutor,
+        toolDiscoveryProvider: provider,
+      });
+
+      const result = yield* engine.execute(
+        [
+          "return await tools.search({",
+          '  query: "calendar events",',
+          '  namespace: "calendar",',
+          "  limit: 7,",
+          "  offset: 2,",
+          "});",
+        ].join("\n"),
+        { onElicitation: acceptAll },
+      );
+
+      expect(result.error).toBeUndefined();
+      expect(result.result).toEqual({
+        items: [
+          {
+            path: "custom.searchResult",
+            name: "searchResult",
+            description: "Provided by the host",
+            sourceId: "custom",
+            score: 999,
+          },
+        ],
+        total: 1,
+        hasMore: false,
+        nextOffset: null,
+      });
+      expect(calls).toEqual([
+        {
+          query: "calendar events",
+          namespace: "calendar",
+          limit: 7,
+          offset: 2,
+        },
+      ]);
+    }),
+  );
+
   it.effect("supports executor-scoped source listing and tool search", () =>
     Effect.gen(function* () {
       const executor = yield* makeSearchExecutor();

packages/core/execution/src/tool-invoker.ts

@@ -249,6 +249,20 @@ export type ExecutorSourceListItem = {
   readonly toolCount: number;
 };
 
+export type ToolDiscoveryInput = {
+  readonly executor: Executor;
+  readonly query: string;
+  readonly namespace?: string;
+  readonly limit: number;
+  readonly offset: number;
+};
+
+export interface ToolDiscoveryProvider {
+  readonly searchTools: (
+    input: ToolDiscoveryInput,
+  ) => Effect.Effect<PagedResult<ToolDiscoveryResult>, ExecutionToolError>;
+}
+
 /**
  * Page of results from a list-style discovery tool. Shared by
  * `tools.search` and `tools.executor.sources.list` so the model sees one
@@ -515,6 +529,11 @@ export const searchTools = Effect.fn("executor.tools.search")(function* (
   return page;
 });
 
+export const defaultToolDiscoveryProvider: ToolDiscoveryProvider = {
+  searchTools: ({ executor, query, namespace, limit, offset }) =>
+    searchTools(executor, query, limit, { namespace, offset }),
+};
+
 /** What `tools.executor.sources.list()` calls inside the sandbox. */
 export const listExecutorSources = Effect.fn("executor.sources.list")(function* (
   executor: Executor,