Add the host-agnostic ingest pipeline as @corbits/granola/ingest

.github/workflows/test.yml

@@ -49,6 +49,16 @@ jobs:
       - run: bun install --frozen-lockfile
       - run: bun run test:ingress
 
+  test-ingest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+      - run: bun install --frozen-lockfile
+      - run: bun run test:ingest
+
   build:
     runs-on: ubuntu-latest
     steps:

ARCHITECTURE.md

@@ -1,17 +1,23 @@
 # Architecture
 
-## Two faces, one dependency direction
+## Three faces, one dependency direction
 
-This package has two independent faces, each with its own source directory:
+This package has three independent faces, each with its own source directory:
 
 - `src/tools/` — the Granola API client and the tools an agent calls (fetch a note,
   search notes). Published as the package root, `@corbits/granola`.
 - `src/ingress/` — the extension that receives Granola webhooks, verifies signatures,
   and dispatches notes to handlers. Published as the subpath export
   `@corbits/granola/ingress`.
-
-`src/ingress` depends on `src/tools` for the client. `src/tools` must never depend on
-`src/ingress`, or on any hub, mounting, extension or webhook machinery at all — the
+- `src/ingest/` — the host-agnostic pipeline that turns an acked webhook event into a
+  persisted transcript and a dispatched bucket handler: fetch note, resolve bucket,
+  persist, capture knowledge, hand off. Published as the subpath export
+  `@corbits/granola/ingest`.
+
+`src/ingress` depends on `src/tools` for the client. `src/ingest` depends on both —
+`src/tools` for the client and note shapes, `src/ingress` for the binding store and
+webhook payload it processes. `src/tools` must never depend on `src/ingress` or
+`src/ingest`, or on any hub, mounting, extension or webhook machinery at all — the
 operator requirement is that someone can import the tools and grant them to any agent
 like any other plain Interchange tool, and nothing hub-shaped comes along for the ride.
 

CONTRIBUTING.md

@@ -18,13 +18,14 @@ extension depends on the tools, not the other way around. Checked by
 - **Red first.** A bug fix starts with a test that fails for the reason you believe, and
   you should watch it fail.
 - Assert **behavior a consumer can observe** over internal call shapes.
-- Tools tests live under `src/tools`; ingress tests live under `src/ingress`. Keep that
-  split — it is what lets `test:tools` and `test:ingress` run and fail independently in
-  CI.
+- Tools tests live under `src/tools`; ingress tests live under `src/ingress`; ingest
+  pipeline tests live under `src/ingest`. Keep that split — it is what lets
+  `test:tools`, `test:ingress`, and `test:ingest` run and fail independently in CI.
 
 ## Pull requests
 
 - Keep commits focused, and keep the diff to the change you are describing.
 - Explain *why* in the commit message; the code already says what.
-- CI must be green: dependency check, typecheck, tools tests, ingress tests, and build.
+- CI must be green: dependency check, typecheck, tools tests, ingress tests, ingest
+  tests, and build.
 - Contributions are accepted under the repository's LGPL-2.1-only licence.

package.json

@@ -37,6 +37,11 @@
       "bun": "./src/ingress/index.ts",
       "types": "./dist/ingress/index.d.ts",
       "default": "./dist/ingress/index.js"
+    },
+    "./ingest": {
+      "bun": "./src/ingest/index.ts",
+      "types": "./dist/ingest/index.d.ts",
+      "default": "./dist/ingest/index.js"
     }
   },
   "files": [
@@ -54,6 +59,7 @@
     "check-deps": "bun run scripts/check-deps.ts",
     "test:tools": "bun test src/tools",
     "test:ingress": "bun test src/ingress",
+    "test:ingest": "bun test src/ingest",
     "test": "bun test src",
     "test:coverage": "bun test --coverage src"
   },

src/ingest/bucket-handler.ts

@@ -0,0 +1,47 @@
+/**
+ * The contract a Granola bucket-type handler must satisfy.
+ *
+ * Declared by the pipeline module, same inversion as the lifecycle
+ * contract: the pipeline is the generic caller, so the shape it dispatches
+ * through belongs to whoever implements it. Hosts register concrete
+ * handlers per bucket type (e.g. Scout's diligence and internal handlers).
+ *
+ * `TRef` is whatever the host's transcript store returns from `persist` —
+ * opaque to the pipeline, handed through to the handler unexamined.
+ */
+import type { GranolaBucket, GranolaNote } from "../tools/types.js";
+
+/** Where the ack message for a note's processing run lives; every later post about that note threads under it. */
+export type GranolaThreadAnchor = { channel: string; ts: string };
+
+/**
+ * Context handed to a bucket-type handler once a note's transcript has been
+ * persisted and captured. `threadAnchor` is populated for every real
+ * ingest-triggered dispatch (the ack posted before this handler runs) — it
+ * is optional only so handlers keep working standalone in tests/call sites
+ * that construct a context with no ack thread to reply into.
+ *
+ * `TAnchor` mirrors the pipeline's own anchor type (established by
+ * `GranolaIngestLifecycle.onProcessingStarted`) — defaulted to
+ * `GranolaThreadAnchor` so existing single-type-argument call sites keep
+ * compiling unchanged.
+ */
+export type GranolaBucketHandlerContext<
+  TRef = unknown,
+  TAnchor = GranolaThreadAnchor,
+> = {
+  note: GranolaNote;
+  transcriptText: string;
+  bucket: GranolaBucket;
+  artifactRef: TRef;
+  threadAnchor?: TAnchor;
+  /**
+   * Set by a host's re-entry affordance (e.g. Scout's ambiguous-ask card):
+   * the handler pins these companies on the run, skipping extraction.
+   */
+  pinnedCompanies?: string[];
+};
+
+export type GranolaBucketHandler<TRef = unknown, TAnchor = GranolaThreadAnchor> = (
+  context: GranolaBucketHandlerContext<TRef, TAnchor>,
+) => Promise<void>;

src/ingest/index.ts

@@ -0,0 +1,20 @@
+/**
+ * Public surface of the Granola ingest pipeline: the chat- and
+ * host-agnostic machinery downstream of an acked webhook event. Hosts
+ * provide the lifecycle (rendering), transcript store (persistence),
+ * knowledge capture (enrichment), and bucket handlers (behavior).
+ */
+export { createGranolaIngest } from "./pipeline.js";
+export type {
+  CreateGranolaIngestOptions,
+  GranolaIngest,
+  GranolaIngestHandlers,
+  GranolaKnowledgeCapture,
+  GranolaTranscriptStore,
+} from "./pipeline.js";
+export type { GranolaIngestLifecycle } from "./lifecycle.js";
+export type {
+  GranolaBucketHandler,
+  GranolaBucketHandlerContext,
+  GranolaThreadAnchor,
+} from "./bucket-handler.js";

src/ingest/lifecycle.ts

@@ -0,0 +1,115 @@
+/**
+ * The lifecycle-hooks CONTRACT the Granola ingest pipeline reports through.
+ *
+ * Declared by the pipeline module because the pipeline is the generic
+ * caller: the shape it reports through belongs to whoever implements it.
+ * A host provides a concrete implementation (e.g. Scout's Slack-rendering
+ * lifecycle) and hands it to `createGranolaIngest`.
+ */
+import type { GranolaBucketType } from "../tools/types.js";
+import type { GranolaThreadAnchor } from "./bucket-handler.js";
+
+/**
+ * Every human-visible event the pipeline reports, in the order a single
+ * note's run can produce them. `TAnchor` is whatever the host's
+ * `onProcessingStarted` returns — opaque to the pipeline, threaded back
+ * unexamined through every later hook for that note's run.
+ *
+ * One hook per event, not one generic "post text" hook: each carries
+ * exactly the structured data a host needs to render its own copy (or
+ * choose not to post at all), rather than the pipeline deciding wording.
+ */
+export type GranolaIngestLifecycle<TAnchor = GranolaThreadAnchor> = {
+  /**
+   * A second event for a note already being processed arrived — usually
+   * two webhook deliveries racing (e.g. `note.generated` and `note.edited`
+   * close together). `anchor` is set once processing reached
+   * `onProcessingStarted`; `bucketChannel` is set once a bucket resolved
+   * even if processing hasn't reached that point yet. Both undefined means
+   * there is nowhere known to notify — the host may no-op.
+   */
+  onDuplicateEvent(args: {
+    noteId: string;
+    anchor: TAnchor | undefined;
+    bucketChannel: string | undefined;
+  }): Promise<void>;
+
+  /**
+   * The note couldn't be fetched from the Granola API at all — no bucket is
+   * known yet (folder membership only comes back from the fetch that just
+   * failed), so `candidateChannels` is every channel any current binding
+   * points at, deduped. The host decides how to broadcast (or not) across
+   * them.
+   */
+  onFetchFailed(args: {
+    noteId: string;
+    error: string;
+    candidateChannels: string[];
+  }): Promise<void>;
+
+  /**
+   * A `note.access_granted` event arrived for a note whose AI summary hasn't
+   * been generated yet — Granola's API 404s on ungenerated notes and offers
+   * no way to trigger generation. Recovery is automatic: a human clicking
+   * "Generate Notes" in Granola fires `note.generated`, which re-enters the
+   * pipeline. Like `onFetchFailed`, no bucket is known yet, so
+   * `candidateChannels` is every bound channel.
+   */
+  onNoteNotGenerated(args: {
+    noteId: string;
+    candidateChannels: string[];
+  }): Promise<void>;
+
+  /**
+   * A re-added note already has a persisted transcript — the durable
+   * human-in-the-loop gate: a human decides whether to reprocess rather
+   * than the pipeline silently skipping or blindly rerunning.
+   */
+  onAlreadyProcessed(args: {
+    noteId: string;
+    noteTitle: string;
+    bucketChannel: string;
+  }): Promise<void>;
+
+  /**
+   * Fresh processing is starting for a note whose bucket just resolved.
+   * Returns the anchor this note's run threads every later hook through —
+   * e.g. a Slack host posts an initial message here and returns its
+   * channel+ts.
+   */
+  onProcessingStarted(args: {
+    noteId: string;
+    noteTitle: string;
+    bucketChannel: string;
+  }): Promise<TAnchor>;
+
+  /**
+   * The note has no transcript/summary yet. `isFinalAttempt: false` means a
+   * retry was just scheduled `retryDelayMs` out; `true` means the retry
+   * already ran and the pipeline is giving up on the note until another
+   * webhook event arrives for it.
+   */
+  onNoteNotReady(args: {
+    anchor: TAnchor;
+    isFinalAttempt: boolean;
+    retryDelayMs: number;
+  }): Promise<void>;
+
+  /** Transcript/summary text is in hand and about to be persisted. */
+  onTranscriptReady(args: {
+    anchor: TAnchor;
+    segmentCount: number | undefined;
+  }): Promise<void>;
+
+  /** Persisting the transcript artifact failed; processing stops here. */
+  onPersistFailed(args: { anchor: TAnchor; error: string }): Promise<void>;
+
+  /** About to dispatch to the bucket-type handler — the last stage before handler-specific behavior takes over. */
+  onHandlerDispatching(args: {
+    anchor: TAnchor;
+    bucketType: GranolaBucketType;
+  }): Promise<void>;
+
+  /** The bucket-type handler threw. */
+  onHandlerFailed(args: { anchor: TAnchor; error: string }): Promise<void>;
+};