From 5a441a52c0fb8bee240c7352c465501c5c2b3061 Mon Sep 17 00:00:00 2001 From: psteinroe Date: Mon, 14 Sep 2026 23:54:11 +0000 Subject: [PATCH 1/3] feat(events): add durable custom event pipeline --- README.md | 2 +- docs/content/api/conductor.md | 2 - docs/content/api/orchestrator.md | 8 +- docs/content/crafting-tasks/defining-tasks.md | 65 +- docs/content/crafting-tasks/triggers.md | 153 +--- docs/content/index.md | 2 +- docs/content/scaling/maintenance.md | 3 +- implementation-notes.md | 42 + migrations/0000000001_setup.sql | 529 +++++++++-- migrations/0000000002_events.sql | 346 -------- packages/pgconductor-js/src/conductor.ts | 73 +- .../pgconductor-js/src/database-client.ts | 52 +- .../pgconductor-js/src/event-definition.ts | 106 +-- .../src/event-trigger-validation.ts | 184 ++++ packages/pgconductor-js/src/generated/sql.ts | 819 +++++++++--------- .../pgconductor-js/src/maintenance-task.ts | 14 +- packages/pgconductor-js/src/orchestrator.ts | 116 ++- packages/pgconductor-js/src/query-builder.ts | 44 +- packages/pgconductor-js/src/schemas.ts | 71 +- packages/pgconductor-js/src/select-columns.ts | 34 +- packages/pgconductor-js/src/task-context.ts | 8 +- .../pgconductor-js/src/task-definition.ts | 110 +-- packages/pgconductor-js/src/task.ts | 90 +- packages/pgconductor-js/src/versions.ts | 2 +- packages/pgconductor-js/src/worker.ts | 246 ++++-- .../tests/integration/event-pipeline.test.ts | 743 ++++++++++++++++ .../tests/integration/event-triggers.test.ts | 365 ++------ .../integration/maintenance-task.test.ts | 1 + .../tests/integration/schema-manager.test.ts | 1 + .../subscription-lifecycle.test.ts | 298 ++----- .../tests/mocks/database-client.mock.ts | 4 +- .../tests/mocks/in-memory-database-client.ts | 207 +++-- .../tests/unit/emit-types.test.ts | 10 +- .../tests/unit/event-pipeline-types.test.ts | 25 + .../tests/unit/event-trigger-types.test.ts | 323 +++---- .../unit/event-trigger-validation.test.ts | 40 + .../tests/unit/task-event-types.test.ts | 150 +--- .../tests/unit/worker-lifecycle.test.ts | 32 + 38 files changed, 2970 insertions(+), 2350 deletions(-) create mode 100644 implementation-notes.md delete mode 100644 migrations/0000000002_events.sql create mode 100644 packages/pgconductor-js/src/event-trigger-validation.ts create mode 100644 packages/pgconductor-js/tests/integration/event-pipeline.test.ts create mode 100644 packages/pgconductor-js/tests/unit/event-pipeline-types.test.ts create mode 100644 packages/pgconductor-js/tests/unit/event-trigger-validation.test.ts create mode 100644 packages/pgconductor-js/tests/unit/worker-lifecycle.test.ts diff --git a/README.md b/README.md index 1b7ab26..5875740 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ **Durable Execution** - Tasks survive crashes and restarts through automatic step memoization -**Multiple Triggers** - Invocable APIs, cron schedules, custom events, and database triggers +**Multiple Triggers** - Invocable APIs, cron schedules, and durable custom events **Workflows** - Invoke child tasks and wait for results with full type safety diff --git a/docs/content/api/conductor.md b/docs/content/api/conductor.md index 1956b1a..3eb14cf 100644 --- a/docs/content/api/conductor.md +++ b/docs/content/api/conductor.md @@ -16,7 +16,6 @@ const conductor = Conductor.create({ context, // Custom context object logger?, // Optional custom logger events?, // Optional EventSchemas - database?, // Optional DatabaseSchema }); ``` @@ -28,7 +27,6 @@ const conductor = Conductor.create({ - `context`: Object passed to task handlers as `ctx.{property}` (see [Custom Context](../crafting-tasks/custom-context.md)) - `logger`: (optional) Custom logger implementation - `events`: (optional) `EventSchemas.fromSchema([...])` for typed custom events -- `database`: (optional) `DatabaseSchema.fromSchema({...})` for database event triggers **Connection options:** diff --git a/docs/content/api/orchestrator.md b/docs/content/api/orchestrator.md index d7b043e..b9e86a2 100644 --- a/docs/content/api/orchestrator.md +++ b/docs/content/api/orchestrator.md @@ -24,6 +24,11 @@ const orchestrator = Orchestrator.create({ - `workers`: (optional) Array of custom workers from `conductor.createWorker()` - `defaultWorker`: (optional) Configuration for default worker +The Orchestrator is the registration authority for its workers. Configure at +most one worker for each queue (including the implicit `default` worker); put +all tasks for a queue on that worker. Multiple workers for one queue are +rejected rather than competing for registrations. + **Default worker config:** ```typescript @@ -71,7 +76,8 @@ await orchestrator.drain(); ``` - Starts the orchestrator -- Processes until queue is empty +- Processes until all queues are globally quiescent, including event fan-out + created while another queue is finishing - Automatically stops - Returns when all work is done diff --git a/docs/content/crafting-tasks/defining-tasks.md b/docs/content/crafting-tasks/defining-tasks.md index 4e06f6d..b4ef20e 100644 --- a/docs/content/crafting-tasks/defining-tasks.md +++ b/docs/content/crafting-tasks/defining-tasks.md @@ -1,6 +1,6 @@ # Defining Tasks and Events -Define type-safe tasks, events, and database schemas that work across your entire application. +Define type-safe tasks and events that work across your entire application. ## Why Define Schemas? @@ -151,60 +151,6 @@ const conductor = Conductor.create({ }); ``` -## Database Schema - -Database schemas enable type-safe database triggers. These are always TypeScript types generated from your database. - -### Generating Types - -Use your preferred tool to generate types from your database: - -**With pgconductor CLI:** - -```bash -pnpx pgconductor gen types typescript --db-url "postgres://localhost/mydb" --schemas "public" > database.types.ts -``` - -**With Supabase CLI:** - -```bash -pnpx supabase gen types typescript --linked > database.types.ts -``` - -### Registering Database Schema - -```typescript -import { DatabaseSchema } from "pgconductor-js"; -import type { Database } from "./database.types"; - -const conductor = Conductor.create({ - connectionString: "postgres://localhost/mydb", - tasks: TaskSchemas.fromSchema([...]), - database: DatabaseSchema.fromGeneratedTypes(), - // or if you are using Supabase CLI - // database: DatabaseSchema.fromSupabaseTypes(), - context: {}, -}); -``` - -Now database triggers are fully typed: - -```typescript -conductor.createTask( - { name: "on-user-created" }, - { - schema: "public", - table: "users", - operation: "insert", - columns: "id,email,name", // TypeScript validates these columns exist! - }, - async (event, ctx) => { - // event.payload.new is typed as { id: string, email: string, name: string } - const { id, email, name } = event.payload.new; - } -); -``` - ## Return Types Tasks can return typed results when invoked from other tasks using `ctx.invoke()`: @@ -264,8 +210,7 @@ packages/ ├── schemas/ # Shared schema package │ ├── package.json │ ├── tasks.ts # Task definitions -│ ├── events.ts # Event definitions -│ └── database.types.ts # Generated database types +│ └── events.ts # Event definitions ├── worker/ # Worker service │ ├── package.json │ └── src/ @@ -328,16 +273,14 @@ export const userCreated = defineEvent({ `worker/src/index.ts`: ```typescript -import { Conductor, Orchestrator, TaskSchemas, EventSchemas, DatabaseSchema } from "pgconductor-js"; +import { Conductor, Orchestrator, TaskSchemas, EventSchemas } from "pgconductor-js"; import { sendEmailTask, processOrderTask } from "@myapp/schemas/tasks"; import { userCreated } from "@myapp/schemas/events"; -import type { Database } from "@myapp/schemas/database.types"; const conductor = Conductor.create({ connectionString: process.env.DATABASE_URL, tasks: TaskSchemas.fromSchema([sendEmailTask, processOrderTask]), events: EventSchemas.fromSchema([userCreated]), - database: DatabaseSchema.fromGeneratedTypes(), context: {}, }); @@ -407,6 +350,6 @@ await conductor.emit("user.created", { ## What's Next? -- [Task Triggers](triggers.md) - Configure invocable, cron, event, and database triggers +- [Task Triggers](triggers.md) - Configure invocable, cron, and custom event triggers - [Testing](testing.md) - Unit test tasks with type-safe mocks - [Conductor API](../api/conductor.md) - Full API documentation diff --git a/docs/content/crafting-tasks/triggers.md b/docs/content/crafting-tasks/triggers.md index b4de1f2..d13eb5b 100644 --- a/docs/content/crafting-tasks/triggers.md +++ b/docs/content/crafting-tasks/triggers.md @@ -5,7 +5,6 @@ Tasks can be triggered in multiple ways: - **Invocable** - Triggered manually via `conductor.invoke()` - **Cron** - Triggered on a schedule - **Custom Events** - Triggered when you emit custom application events -- **Database Triggers** - Triggered automatically by Postgres triggers on INSERT/UPDATE/DELETE ## Invocable Tasks @@ -109,139 +108,70 @@ await conductor.emit("user.created", { }); ``` -### Field Selection +Each event is appended to the event log and paired with a short-lived internal dispatch execution. Destination fan-out and the event's `dispatched_at` state commit in one database transaction, so failures retry without duplicating destination executions. Event rows are retained independently for seven days by default. -For large events, you can select only specific fields to reduce payload size: +### Emitting from Database Triggers -```typescript -conductor.createTask( - { name: "log-user-id" }, - { event: "user.created", fields: "userId" }, - async (event, ctx) => { - // event.payload only contains { userId: string } - ctx.logger.info("User created:", event.payload.userId); - } -); -``` +Postgres Conductor does not create or manage triggers on application tables. If a database change should emit an event, define the trigger in your own migrations and call `pgconductor.emit_event()`: -## Database Triggers +```sql +create function app.emit_user_created() +returns trigger +language plpgsql +as $$ +begin + perform pgconductor.emit_event('user.created', to_jsonb(new)); + return new; +end; +$$; -React to database changes automatically using Postgres triggers. When a row is inserted, updated, or deleted, Postgres Conductor creates a task execution with the row data. +create trigger emit_user_created +after insert on app.users +for each row execute function app.emit_user_created(); +``` -### Setup +Because `emit_event()` inserts both the event row and its dispatch execution in the current transaction, emission is committed or rolled back with the database change. -Provide your database schema types to enable type-safe database triggers: +### Event Filters -```typescript -import { DatabaseSchema } from "pgconductor-js"; -import type { Database } from "./database.types"; // Generated types +Declare top-level scalar fields as filterable, then register equality allowlists on handlers: -const conductor = Conductor.create({ - connectionString: "postgres://localhost/mydb", - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, +```typescript +const orderChanged = defineEvent({ + name: "order.changed", + payload: z.object({ status: z.string(), region: z.string() }), + filterable: ["status", "region"], }); -``` - -### Creating Database Triggers -```typescript -const onContactInsert = conductor.createTask( - { name: "on-contact-insert" }, +conductor.createTask( + { name: "handle-paid-us-orders" }, { - schema: "public", - table: "contact", - operation: "insert", - columns: "id,email,first_name", // Required + event: "order.changed", + filter: { status: ["paid", "trial"], region: ["us"] }, }, async (event, ctx) => { - // event.name === "public.contact.insert" - // event.payload.tg_op === "INSERT" - // event.payload.old === null (no old row on insert) - // event.payload.new contains selected columns - - const { id, email, first_name } = event.payload.new; - ctx.logger.info(`New contact: ${first_name} (${email})`); + // Both fields matched; values within one field are alternatives. } ); ``` -The orchestrator automatically creates Postgres triggers on your tables when it starts. +Fields are combined with AND and values within a field with OR. Strings, numbers, booleans, and `null` are type-sensitive; a missing field does not match `null`. A filter may contain up to 8 fields and 4 values per field. -### Operations - -Support for INSERT, UPDATE, and DELETE: - -**INSERT** - Only `new` row available: -```typescript -{ - schema: "public", - table: "contact", - operation: "insert", - columns: "id,email" -} -// event.payload.old === null -// event.payload.new === { id: string, email: string | null } -``` - -**UPDATE** - Both `old` and `new` rows available: -```typescript -{ - schema: "public", - table: "contact", - operation: "update", - columns: "id,email,name" -} -// event.payload.old === { id: string, email: string | null, name: string } -// event.payload.new === { id: string, email: string | null, name: string } -``` - -**DELETE** - Only `old` row available: -```typescript -{ - schema: "public", - table: "contact", - operation: "delete", - columns: "id,email" -} -// event.payload.old === { id: string, email: string | null } -// event.payload.new === null -``` - -### Conditional Triggers +### Field Selection -Use `when` clause to filter which rows trigger task execution: +For large events, you can select only specific fields to reduce payload size: ```typescript conductor.createTask( - { name: "on-active-user" }, - { - schema: "public", - table: "users", - operation: "insert", - columns: "id,email,name", - when: "NEW.active = true", // Only trigger for active users - }, + { name: "log-user-id" }, + { event: "user.created", fields: "userId" }, async (event, ctx) => { - // Only called when active = true + // event.payload only contains { userId: string } + ctx.logger.info("User created:", event.payload.userId); } ); ``` -The `when` clause is evaluated in the Postgres trigger before creating a task execution. - -### How It Works - -When you start the orchestrator: - -1. Postgres triggers are created on your specified tables -2. When a row changes, the trigger captures the row data -3. A task execution is created with the selected columns -4. Your task handler receives the event with typed payload - -The triggers persist in the database even after the orchestrator stops. - ## Multiple Triggers Tasks can respond to multiple trigger types: @@ -253,12 +183,6 @@ const flexibleTask = conductor.createTask( { invocable: true }, { cron: "0 * * * *", name: "hourly" }, { event: "user.created" }, - { - schema: "public", - table: "contact", - operation: "insert", - columns: "id,email" - }, ], async (event, ctx) => { // Discriminate based on event.name @@ -270,9 +194,6 @@ const flexibleTask = conductor.createTask( } else if (event.name === "user.created") { // Custom event const { userId, email, name } = event.payload; - } else if (event.name === "public.contact.insert") { - // Database trigger - const { id, email } = event.payload.new; } } ); diff --git a/docs/content/index.md b/docs/content/index.md index cbf1830..76f8afa 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -14,7 +14,7 @@ **Durable Execution** - Tasks survive crashes and restarts through automatic step memoization -**Multiple Triggers** - Invocable APIs, cron schedules, custom events, and database triggers +**Multiple Triggers** - Invocable APIs, cron schedules, and durable custom events **Workflows** - Invoke child tasks and wait for results with full type safety diff --git a/docs/content/scaling/maintenance.md b/docs/content/scaling/maintenance.md index 00299cb..56f22e9 100644 --- a/docs/content/scaling/maintenance.md +++ b/docs/content/scaling/maintenance.md @@ -8,8 +8,9 @@ Maintenance runs daily on each queue: - Removes old completed executions (based on task retention config) - Removes old failed executions (based on task retention config) +- On the always-present internal queue, removes settled custom-event log rows after seven days -No manual intervention required. +Event cleanup is independent of execution retention. It is bounded and skips events whose internal dispatch execution may still be retried. ## What's Next? diff --git a/implementation-notes.md b/implementation-notes.md new file mode 100644 index 0000000..0b8c5de --- /dev/null +++ b/implementation-notes.md @@ -0,0 +1,42 @@ +# Implementation Notes + +Running notes on how the event-pipeline rewrite interprets the agreed design. + +## Design decisions + +- Custom events are immutable rows in `_private_custom_events`. The row ID is also the ID of a short-lived execution on the reserved `pgconductor.internal` queue for task `pgconductor.event-dispatch`. +- The event row is the durable event body and replay source. The internal execution provides scheduling, claim fencing, retry, settlement, drain, and shutdown behavior only; deleting that execution does not delete the event. +- A hidden worker uses the ordinary execution lifecycle. User workers register first so the first successful dispatch sees a complete committed subscription snapshot. +- Emission performs only the event-log and dispatch-execution inserts. It does not take a per-key advisory lock, so unrelated application transactions and hot event keys are not serialized for a downstream feature that is not implemented in this PR. +- Fan-out is claim-fenced and atomically inserts destinations and sets `dispatched_at`. A retry of a committed dispatch returns the already-dispatched event, including zero-match events. +- Event destinations use dedicated `source_event_id` and `event_subscription_id` columns. They do not use `parent_execution_id`, so workflow child/orphan semantics remain separate. +- Persistent task subscriptions use two tables: one canonical filter row and exact scalar predicate rows for every field alternative. Dispatch expands scalar event fields, probes the exact-predicate index, and keeps subscriptions whose distinct matched fields equal their stored field count. The normalized predicates are authoritative; dispatch does not re-evaluate the canonical JSON filter. +- Filter semantics are AND across fields and OR across alternatives. Equality is scalar and type-sensitive; a missing field differs from JSON `null`; `{}` is unconditional. +- TypeScript performs catalog and policy validation, canonical field ordering, and type-sensitive alternative deduplication. PostgreSQL derives `field_count` from the stored filter and retains row-shape, scalar, byte-size, and transactional integrity checks. +- Registrations lock worker and dead-letter queues in one global order, then call a dedicated queue-scoped subscription replacement function. Replacement and exact-predicate compilation are set-based; route locks, route-wide subscription counts, and procedural group/clause compilation were removed. +- Managed database-trigger subscriptions remain out of scope. Applications own PostgreSQL triggers and call `pgconductor.emit_event()` transactionally. +- Dispatched events have a seven-day default retention policy. The always-present internal worker owns global cleanup, including named-queue-only deployments. Cleanup is bounded and skips any event whose internal dispatch execution is still active, preventing cleanup from racing recovery after fan-out committed but source settlement did not. + +## Benchmark evidence + +Throwaway PostgreSQL benchmarks compared the previous normalized four-table matcher with two simpler designs at 10,000 subscriptions per event key, batches of 1 and 10, selective, broad, and unfiltered workloads, plus selective 100,000-subscription evidence. + +- A single subscription row plus residual JSONB matching was rejected: selective batch-10 matching regressed from roughly 6 ms to roughly 937 ms. +- An anchor row plus residual JSONB matching improved registration but remained about twice as slow in matching and was rejected. +- The selected flattened exact matcher reduced registration to roughly 43–68% of normalized time. Selective batch-10 matching was roughly 15–19 ms versus roughly 12 ms normalized; broad batch-10 matching was roughly twice as fast, and unfiltered matching roughly four to six times faster. +- At 100,000 subscriptions, flattened selective matching remained bounded rather than exhibiting the full-scan behavior of the residual-only model. + +These measurements are architecture evidence, not throughput or capacity claims. All benchmark code and generated artifacts were removed. + +## Deviations and tradeoffs + +- Migrations are rewritten in place under the repository's active-development policy. Databases created from earlier unreleased migration snapshots must be recreated; this PR intentionally does not add a compatibility migration. +- The schema supports only the scalar equality semantics exposed by the typed API. Prefix, range, negative, and fallback operators were intentionally not added. +- `LISTEN/NOTIFY` was not added. Persisted executions and polling remain authoritative. +- Dispatcher batches remain 10. Fan-out is one atomic transaction per batch; this bounds ordinary retry amplification without introducing chunk-progress state. +- Indexed filter values are limited to 1,024 UTF-8 bytes, event keys to 255 bytes, and field names to 128 bytes so the composite B-tree index cannot exceed PostgreSQL tuple limits. +- Event positions may contain gaps and do not imply commit order. This PR intentionally does not impose a `waitForEvent` linearization protocol on every emitter. + +## Downstream `waitForEvent` + +Reusable primitives are the append-only event log, canonical filter representation, immutable matcher, and exact-predicate indexing pattern. One-shot waits still require a separate subscription table, timeout/cancellation lifecycle, and an explicit no-miss registration boundary; persistent task subscriptions and their queue-snapshot replacement function are not overloaded for that purpose. diff --git a/migrations/0000000001_setup.sql b/migrations/0000000001_setup.sql index 01a65df..b81f856 100644 --- a/migrations/0000000001_setup.sql +++ b/migrations/0000000001_setup.sql @@ -278,7 +278,11 @@ begin RETURN NEW; elsif tg_op = 'UPDATE' then - -- protect default queue from modification + -- protect queues required by the runtime + if old.name = 'pgconductor.internal' or new.name = 'pgconductor.internal' then + raise exception 'Modifying the internal queue is not allowed'; + end if; + if old.name = 'default' or new.name = 'default' then raise exception 'Modifying the default queue is not allowed'; end if; @@ -291,7 +295,11 @@ begin return new; elsif tg_op = 'DELETE' then - -- protect default queue from deletion + -- protect queues required by the runtime + if old.name = 'pgconductor.internal' then + raise exception 'Deleting the internal queue is not allowed'; + end if; + if old.name = 'default' then raise exception 'Deleting the default queue is not allowed'; end if; @@ -355,22 +363,11 @@ create type pgconductor.task_spec as ( dead_letter_task_key text ); -create type pgconductor._private_event_operation as enum ( - 'insert', - 'update', - 'delete' -); - create type pgconductor.event_subscription_spec as ( task_key text, - queue text, event_key text, - schema_name text, - table_name text, - operation pgconductor._private_event_operation, - when_clause text, payload_fields text[], - column_names text[] + filter jsonb ); create or replace function pgconductor._private_register_worker( @@ -385,19 +382,35 @@ volatile set search_path to '' as $function$ begin - -- step 1: upsert queue (triggers partition creation) + -- Upsert every required queue in a stable order (triggers partition + -- creation). Registrations may reference one another as dead-letter queues, + -- so queue creation and locking share a global order. insert into pgconductor._private_queues (name) - values (p_queue_name) + select required.name + from ( + select p_queue_name as name + union + select spec.dead_letter_queue + from unnest(p_task_specs) as spec + where spec.dead_letter_queue is not null + ) required + order by required.name on conflict (name) do nothing; - -- Dead-letter destinations may not have a worker yet; create their partitions. - insert into pgconductor._private_queues (name) - select distinct spec.dead_letter_queue - from unnest(p_task_specs) as spec - where spec.dead_letter_queue is not null - on conflict (name) do nothing; + -- Queue row locks are the only registration locks; acquire them in order. + perform 1 + from pgconductor._private_queues queue + where queue.name in ( + select p_queue_name + union + select spec.dead_letter_queue + from unnest(p_task_specs) as spec + where spec.dead_letter_queue is not null + ) + order by queue.name + for update; - -- step 2: register/update tasks + -- Register/update tasks. insert into pgconductor._private_tasks (key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days, window_start, window_end, concurrency_limit, group_concurrency_limit, dead_letter_queue, dead_letter_task_key) select spec.key, @@ -425,7 +438,7 @@ begin dead_letter_queue = excluded.dead_letter_queue, dead_letter_task_key = excluded.dead_letter_task_key; - -- step 3: insert scheduled cron executions + -- Insert scheduled cron executions. insert into pgconductor._private_executions (task_key, queue, payload, run_at, dedupe_key, cron_expression, "group") select spec.task_key, @@ -443,8 +456,7 @@ begin cron_expression = excluded.cron_expression, "group" = excluded."group"; - -- step 4: clean up stale schedules for this queue - -- delete future executions for schedules that no longer exist + -- Clean up stale schedules for this queue. delete from pgconductor._private_executions where queue = p_queue_name and cron_expression is not null @@ -456,7 +468,6 @@ begin where spec.dedupe_key is not null and spec.dedupe_key like 'scheduled::%' ); - -- mark running executions as cancelled for schedules that no longer exist update pgconductor._private_executions set cancelled = true where queue = p_queue_name @@ -472,59 +483,10 @@ begin where spec.dedupe_key is not null and spec.dedupe_key like 'scheduled::%' ); - -- step 5: manage event subscriptions (only recreate triggers when subscriptions change) - merge into pgconductor._private_event_subscriptions as target - using ( - select - s.task_key, - s.queue, - s.event_key, - s.schema_name, - s.table_name, - s.operation, - s.when_clause, - s.payload_fields, - s.column_names - from unnest(p_event_subscriptions) as s - ) as source - on ( - target.queue = source.queue and - target.task_key = source.task_key and - coalesce(target.event_key, '') = coalesce(source.event_key, '') and - coalesce(target.schema_name, '') = coalesce(source.schema_name, '') and - coalesce(target.table_name, '') = coalesce(source.table_name, '') and - coalesce(target.operation::text, '') = coalesce(source.operation::text, '') and - coalesce(target.when_clause, '') = coalesce(source.when_clause, '') and - coalesce(array_to_string(target.payload_fields, ','), '') = - coalesce(array_to_string(source.payload_fields, ','), '') and - coalesce(array_to_string(target.column_names, ','), '') = - coalesce(array_to_string(source.column_names, ','), '') - ) - when not matched then insert ( - task_key, queue, event_key, schema_name, table_name, operation, - when_clause, payload_fields, column_names - ) values ( - source.task_key, source.queue, source.event_key, - source.schema_name, source.table_name, source.operation, - source.when_clause, source.payload_fields, source.column_names + perform pgconductor._private_replace_custom_event_subscriptions( + p_queue_name, + p_event_subscriptions ); - - -- step 6: delete old subscriptions for this queue not in new set - delete from pgconductor._private_event_subscriptions target - where target.queue = p_queue_name - and not exists ( - select 1 from unnest(p_event_subscriptions) source - where target.task_key = source.task_key - and coalesce(target.event_key, '') = coalesce(source.event_key, '') - and coalesce(target.schema_name, '') = coalesce(source.schema_name, '') - and coalesce(target.table_name, '') = coalesce(source.table_name, '') - and coalesce(target.operation::text, '') = coalesce(source.operation::text, '') - and coalesce(target.when_clause, '') = coalesce(source.when_clause, '') - and coalesce(array_to_string(target.payload_fields, ','), '') = - coalesce(array_to_string(source.payload_fields, ','), '') - and coalesce(array_to_string(target.column_names, ','), '') = - coalesce(array_to_string(source.column_names, ','), '') - ); end; $function$; @@ -908,3 +870,412 @@ begin end; $function$; +-- Event deliveries are independent executions. The source execution has the +-- same UUID as its event-log row; destinations carry the event identity and +-- subscription identity in dedicated columns rather than using workflow +-- parentage. +alter table pgconductor._private_executions + add column source_event_id uuid, + add column event_subscription_id uuid, + drop column if exists subscription_id; + +alter table pgconductor._private_executions + add constraint chk_executions_event_delivery_pair + check ((source_event_id is null) = (event_subscription_id is null)); + +alter table pgconductor._private_executions + add constraint chk_executions_event_delivery_no_parent + check (source_event_id is null or parent_execution_id is null); + +create unique index idx_executions_event_destination + on pgconductor._private_executions (source_event_id, event_subscription_id, queue) + where source_event_id is not null and event_subscription_id is not null; + +-- A small immutable helper is used by the generated subscription field count. +create or replace function pgconductor._private_jsonb_object_size(p_value jsonb) +returns integer +language sql +immutable +strict +set search_path to '' +as $function$ + select count(*)::integer from jsonb_object_keys(p_value); +$function$; + +create table pgconductor._private_custom_event_subscriptions ( + id uuid primary key default pgconductor._private_portable_uuidv7(), + event_key text not null, + task_key text not null, + queue text not null, + payload_fields text[], + filter jsonb not null default '{}'::jsonb, + field_count integer generated always as + (pgconductor._private_jsonb_object_size(filter)) stored, + created_at timestamptz not null default pgconductor._private_current_time(), + constraint chk_custom_event_subscription_event_key check ( + btrim(event_key) <> '' and octet_length(event_key) between 1 and 255 + ), + constraint chk_custom_event_subscription_filter_object check ( + jsonb_typeof(filter) = 'object' + ) +); + +create index idx_custom_event_subscriptions_event + on pgconductor._private_custom_event_subscriptions (event_key, id); +create index idx_custom_event_subscriptions_unfiltered + on pgconductor._private_custom_event_subscriptions (event_key, id) + where field_count = 0; + +create table pgconductor._private_custom_event_predicates ( + id bigint generated always as identity primary key, + subscription_id uuid not null references pgconductor._private_custom_event_subscriptions(id) on delete cascade, + event_key text not null, + field_name text not null, + value jsonb not null, + constraint uq_custom_event_predicate_alternative + unique (subscription_id, field_name, value), + constraint chk_custom_event_predicate_event_key check ( + btrim(event_key) <> '' and octet_length(event_key) between 1 and 255 + ), + constraint chk_custom_event_predicate_field_name check ( + btrim(field_name) <> '' and octet_length(field_name) <= 128 + ), + constraint chk_custom_event_predicate_scalar check ( + jsonb_typeof(value) in ('string', 'number', 'boolean', 'null') + ), + constraint chk_custom_event_predicate_scalar_text_size check ( + octet_length(value::text) <= 1024 + ) +); + +create index idx_custom_event_predicates_exact + on pgconductor._private_custom_event_predicates + (event_key, field_name, value, subscription_id); + +-- Persistent worker subscriptions are replaced as one queue-scoped snapshot. +-- Canonicalization and policy validation happen in TypeScript; database checks +-- retain only storage and index integrity guarantees. +create or replace function pgconductor._private_replace_custom_event_subscriptions( + p_queue_name text, + p_subscriptions pgconductor.event_subscription_spec[] +) +returns void +language sql +volatile +set search_path to '' +as $function$ + with removed as ( + delete from pgconductor._private_custom_event_subscriptions + where queue = p_queue_name + ), input as materialized ( + select task_key, event_key, payload_fields, filter, input_ordinal + from unnest(p_subscriptions) with ordinality + as subscription(task_key, event_key, payload_fields, filter, input_ordinal) + ), inserted as ( + insert into pgconductor._private_custom_event_subscriptions ( + id, event_key, task_key, queue, payload_fields, filter + ) + select + pgconductor._private_portable_uuidv7(), + input.event_key, + input.task_key, + p_queue_name, + input.payload_fields, + coalesce(input.filter, '{}'::jsonb) + from input + order by input.input_ordinal + returning id, event_key, filter + ) + insert into pgconductor._private_custom_event_predicates ( + subscription_id, event_key, field_name, value + ) + select + inserted.id, + inserted.event_key, + field.key, + alternative.value + from inserted + cross join lateral jsonb_each(inserted.filter) as field + cross join lateral jsonb_array_elements(field.value) as alternative(value); +$function$; + +create table pgconductor._private_custom_events ( + id uuid primary key, + event_position bigint generated always as identity unique, + event_key text not null, + payload jsonb not null, + created_at timestamptz not null default pgconductor._private_current_time(), + dispatched_at timestamptz, + constraint chk_custom_event_event_key check ( + btrim(event_key) <> '' and octet_length(event_key) between 1 and 255 + ), + constraint chk_custom_event_payload_object check ( + jsonb_typeof(payload) = 'object' + ) +); + +-- event_position is a deterministic replay cursor, not commit order. A future +-- one-shot wait implementation must define its own no-miss registration boundary. +create index idx_custom_events_replay + on pgconductor._private_custom_events (event_key, event_position); +create index idx_custom_events_retention + on pgconductor._private_custom_events (dispatched_at, event_position) + where dispatched_at is not null; +create index idx_custom_events_terminal_cleanup + on pgconductor._private_custom_events (created_at, event_position); + +insert into pgconductor._private_queues (name) +values ('pgconductor.internal') +on conflict do nothing; + +insert into pgconductor._private_tasks ( + key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days +) +values ( + 'pgconductor.event-dispatch', 'pgconductor.internal', 3, 0, null +) +on conflict (queue, key) do update set + max_attempts = excluded.max_attempts, + remove_on_complete_days = excluded.remove_on_complete_days, + remove_on_fail_days = excluded.remove_on_fail_days; + +create or replace function pgconductor._private_extract_event_payload(p_fields text[], p_payload jsonb) +returns jsonb +language sql +immutable +set search_path to '' +as $function$ + select case + when p_fields is null then p_payload + else coalesce( + ( + select jsonb_object_agg(key, p_payload -> key) + from unnest(p_fields) key + where p_payload ? key + ), + '{}'::jsonb + ) + end; +$function$; + +-- Claiming, matching, destination insertion, and marking the source dispatched +-- are one statement. The write barrier makes the UPDATE depend on the +-- destination INSERT even when an event has zero matches. +create or replace function pgconductor._private_dispatch_custom_events( + p_event_ids uuid[], + p_orchestrator_id uuid +) +returns table(event_id uuid) +language sql +volatile +security definer +set search_path to '' +as $function$ + with claimed as materialized ( + select + source.id, + source_event.event_key, + source_event.payload, + source_event.dispatched_at + from pgconductor._private_executions source + join pgconductor._private_custom_events source_event + on source_event.id = source.id + where source.id = any(coalesce(p_event_ids, array[]::uuid[])) + and source.queue = 'pgconductor.internal' + and source.task_key = 'pgconductor.event-dispatch' + and source.locked_by = p_orchestrator_id + and source.completed_at is null + and source.failed_at is null + and not source.cancelled + for update of source, source_event + ), pending as materialized ( + select id, event_key, payload + from claimed + where dispatched_at is null + ), event_values as materialized ( + select + pending.id as event_id, + pending.event_key, + pending.payload, + field.key as field_name, + field.value + from pending + cross join lateral jsonb_each(pending.payload) as field + where jsonb_typeof(field.value) in ('string', 'number', 'boolean', 'null') + ), filtered_candidates as materialized ( + select + event_value.event_id, + event_value.event_key, + event_value.payload, + subscription.id as subscription_id, + subscription.task_key, + subscription.queue, + subscription.payload_fields + from event_values event_value + join pgconductor._private_custom_event_predicates predicate + on predicate.event_key = event_value.event_key + and predicate.field_name = event_value.field_name + and jsonb_typeof(predicate.value) = jsonb_typeof(event_value.value) + and predicate.value = event_value.value + join pgconductor._private_custom_event_subscriptions subscription + on subscription.id = predicate.subscription_id + and subscription.event_key = event_value.event_key + and subscription.field_count > 0 + join pgconductor._private_tasks task + on task.key = subscription.task_key + and task.queue = subscription.queue + group by + event_value.event_id, + event_value.event_key, + event_value.payload, + subscription.id, + subscription.task_key, + subscription.queue, + subscription.payload_fields, + subscription.field_count + having count(distinct predicate.field_name) = subscription.field_count + ), unfiltered_candidates as materialized ( + select + pending.id as event_id, + pending.event_key, + pending.payload, + subscription.id as subscription_id, + subscription.task_key, + subscription.queue, + subscription.payload_fields + from pending + join pgconductor._private_custom_event_subscriptions subscription + on subscription.event_key = pending.event_key + and subscription.field_count = 0 + join pgconductor._private_tasks task + on task.key = subscription.task_key + and task.queue = subscription.queue + ), candidates as materialized ( + select * from filtered_candidates + union all + select * from unfiltered_candidates + ), matches as materialized ( + select distinct + candidate.event_id, + candidate.task_key, + candidate.queue, + candidate.payload_fields, + candidate.event_key, + candidate.payload, + candidate.subscription_id + from candidates candidate + ), inserted_destinations as ( + insert into pgconductor._private_executions ( + id, + task_key, + queue, + payload, + source_event_id, + event_subscription_id + ) + select + pgconductor._private_portable_uuidv7(), + candidate_match.task_key, + candidate_match.queue, + jsonb_build_object( + 'event', candidate_match.event_key, + 'payload', pgconductor._private_extract_event_payload( + candidate_match.payload_fields, candidate_match.payload + ) + ), + candidate_match.event_id, + candidate_match.subscription_id + from matches candidate_match + on conflict (source_event_id, event_subscription_id, queue) + where source_event_id is not null and event_subscription_id is not null + do nothing + returning source_event_id + ), write_barrier as ( + select true as ready from inserted_destinations + union all + select true + where not exists (select 1 from inserted_destinations) + ), updated as ( + update pgconductor._private_custom_events source_event + set dispatched_at = pgconductor._private_current_time() + from pending + cross join write_barrier + where source_event.id = pending.id + and source_event.dispatched_at is null + returning source_event.id + ) + select claimed.id as event_id + from claimed + left join updated + on updated.id = claimed.id + where claimed.dispatched_at is not null + or updated.id is not null; +$function$; + +create or replace function pgconductor.emit_event( + p_event_key text, + p_payload jsonb default '{}'::jsonb +) +returns uuid +language sql +volatile +set search_path to '' +as $function$ + with inserted_event as ( + insert into pgconductor._private_custom_events (id, event_key, payload) + values (pgconductor._private_portable_uuidv7(), p_event_key, coalesce(p_payload, '{}'::jsonb)) + returning id + ), inserted_source as ( + insert into pgconductor._private_executions ( + id, task_key, queue, payload + ) + select + inserted_event.id, + 'pgconductor.event-dispatch', + 'pgconductor.internal', + '{}'::jsonb + from inserted_event + returning id + ) + select id from inserted_source; +$function$; + +create or replace function pgconductor._private_remove_custom_events( + p_before timestamptz, + p_batch integer +) +returns integer +language plpgsql +volatile +security definer +set search_path to '' +as $function$ +declare + v_removed integer; +begin + with candidates as materialized ( + select event.id + from pgconductor._private_custom_events event + left join pgconductor._private_executions source + on source.id = event.id + and source.queue = 'pgconductor.internal' + and source.task_key = 'pgconductor.event-dispatch' + where event.created_at < p_before + and ( + source.id is null + or source.completed_at is not null + or source.failed_at is not null + or source.cancelled + ) + order by event.created_at, event.event_position + limit greatest(coalesce(p_batch, 0), 0) + for update of event skip locked + ) + delete from pgconductor._private_custom_events event + using candidates + where event.id = candidates.id; + + get diagnostics v_removed = row_count; + return v_removed; +end; +$function$; diff --git a/migrations/0000000002_events.sql b/migrations/0000000002_events.sql deleted file mode 100644 index 2d3b223..0000000 --- a/migrations/0000000002_events.sql +++ /dev/null @@ -1,346 +0,0 @@ -alter table pgconductor._private_executions - add column if not exists subscription_id uuid; - -create table if not exists pgconductor._private_custom_events ( - id uuid default pgconductor._private_portable_uuidv7() not null, - event_key text not null, - payload jsonb not null default '{}'::jsonb, - created_at timestamptz default pgconductor._private_current_time() not null, - primary key (created_at, id) -) partition by range (created_at); - -create index if not exists idx_custom_events_event_key - on pgconductor._private_custom_events (event_key, created_at desc); - --- Create initial partition for custom events (will cover many years) -create table if not exists pgconductor._private_custom_events_default - partition of pgconductor._private_custom_events - for values from (minvalue) to (maxvalue); - -create table if not exists pgconductor._private_event_subscriptions ( - id uuid primary key default pgconductor._private_portable_uuidv7(), - - task_key text not null, - queue text not null, - - event_key text, - schema_name text, - table_name text, - operation pgconductor._private_event_operation, - - when_clause text, - payload_fields text[], - column_names text[], - - created_at timestamptz not null default pgconductor._private_current_time(), - - constraint chk_event_type check ( - (event_key is not null and schema_name is null and table_name is null and operation is null) - or - (event_key is null and schema_name is not null and table_name is not null and operation is not null) - ) -); - -create index if not exists idx_event_subscriptions_custom - on pgconductor._private_event_subscriptions (event_key) - where event_key is not null; - -create index if not exists idx_event_subscriptions_database - on pgconductor._private_event_subscriptions (schema_name, table_name, operation) - where schema_name is not null; - -create or replace function pgconductor._private_sync_custom_event_trigger() - returns trigger - language plpgsql - security definer - set search_path to '' -as $_$ -declare - v_invoke_blocks text; - v_has_subscriptions boolean; -begin - -- Only process custom event subscriptions (event_key is not null) - if coalesce(new.event_key, old.event_key) is null then - return coalesce(new, old); - end if; - - drop trigger if exists pgconductor_custom_event on pgconductor._private_custom_events; - drop function if exists pgconductor._private_trigger_custom_event; - - select exists( - select 1 - from pgconductor._private_event_subscriptions - where event_key is not null - ) into v_has_subscriptions; - - if v_has_subscriptions then - v_invoke_blocks := ( - select string_agg(format( - $sql$ - if new.event_key = %L and (%s) then - v_task_keys := array_append(v_task_keys, %L); - v_queues := array_append(v_queues, %L); - v_payloads := array_append(v_payloads, jsonb_build_object('event', new.event_key, 'payload', %s)); - v_subscription_ids := array_append(v_subscription_ids, %L); - end if; - $sql$, - sub.event_key, - coalesce(nullif(sub.when_clause, ''), 'true'), - sub.task_key, - t.queue, - pgconductor._private_build_payload_fields(sub.payload_fields, 'new.payload'), - sub.id - ), e'\n') - from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue - where sub.event_key is not null - ); - - execute format( - $sql$ - create or replace function pgconductor._private_trigger_custom_event() - returns trigger - language plpgsql - security definer - set search_path to '' - as $inner$ - declare - v_task_keys text[]; - v_queues text[]; - v_payloads jsonb[]; - v_subscription_ids uuid[]; - begin - %s - - if array_length(v_task_keys, 1) > 0 then - insert into pgconductor._private_executions (task_key, queue, payload, subscription_id) - select unnest(v_task_keys), unnest(v_queues), unnest(v_payloads), unnest(v_subscription_ids); - end if; - - return new; - end - $inner$ - $sql$, - v_invoke_blocks - ); - - execute $sql$ - create trigger pgconductor_custom_event - after insert on pgconductor._private_custom_events - for each row - execute function pgconductor._private_trigger_custom_event() - $sql$; - end if; - - if tg_op = 'DELETE' then - return old; - end if; - - return new; -end; -$_$; - -create trigger sync_custom_event_trigger - after insert or delete or update on pgconductor._private_event_subscriptions - for each row - execute function pgconductor._private_sync_custom_event_trigger(); - -create or replace function pgconductor._private_build_payload_fields( - p_payload_fields text[], - p_payload_expr text -) - returns text - language sql - immutable - set search_path to '' -as $_$ - select case - when p_payload_fields is null then p_payload_expr - else 'jsonb_build_object(' || array_to_string( - array( - select format('%L, %s->%L', field, p_payload_expr, field) - from unnest(p_payload_fields) as field - ), - ', ' - ) || ')' - end; -$_$; - -create or replace function pgconductor._private_build_column_list( - p_column_names text[], - p_record_name text -) - returns text - language sql - immutable - set search_path to '' -as $_$ - select case - when p_column_names is null then format('row_to_json(%I.*)', p_record_name) - else 'jsonb_build_object(' || array_to_string( - array( - select format('%L, %I.%I', col, p_record_name, col) - from unnest(p_column_names) as col - ), - ', ' - ) || ')' - end; -$_$; - -create or replace function pgconductor._private_sync_database_trigger() - returns trigger - language plpgsql - security definer - set search_path to '' -as $_$ -declare - v_table_name text := coalesce(new.table_name, old.table_name); - v_schema_name text := coalesce(new.schema_name, old.schema_name); - v_op pgconductor._private_event_operation; - v_invoke_blocks text; - v_has_subscriptions boolean; -begin - -- Only process database event subscriptions (schema_name is not null) - if v_schema_name is null then - return coalesce(new, old); - end if; - - -- Process each operation type (insert, update, delete) - foreach v_op in array array['insert', 'update', 'delete']::pgconductor._private_event_operation[] loop - -- Drop existing trigger and function - execute format( - 'drop trigger if exists pgconductor_event_%s on %I.%I', - v_op::text, v_schema_name, v_table_name - ); - - execute format( - 'drop function if exists pgconductor._private_trigger_event_%s_on_%I_%I', - v_op::text, v_schema_name, v_table_name - ); - - -- Check if there are any subscriptions for this operation - select exists( - select 1 - from pgconductor._private_event_subscriptions - where table_name = v_table_name - and schema_name = v_schema_name - and operation = v_op - ) into v_has_subscriptions; - - if v_has_subscriptions then - -- Build if blocks to check conditions and append to arrays - -- Each subscription's when_clause is evaluated inside the trigger function - v_invoke_blocks := ( - select string_agg(format( - $sql$ - if %s then - v_task_keys := array_append(v_task_keys, %L); - v_queues := array_append(v_queues, %L); - v_payloads := array_append(v_payloads, jsonb_build_object( - 'event', %L, - 'payload', jsonb_build_object( - 'old', case when tg_op is distinct from 'INSERT' then %s else null end, - 'new', case when tg_op is distinct from 'DELETE' then %s else null end, - 'tg_table', tg_table_name, - 'tg_op', tg_op - ) - )); - v_subscription_ids := array_append(v_subscription_ids, %L); - end if; - $sql$, - coalesce(nullif(sub.when_clause, ''), 'true'), - sub.task_key, - t.queue, - format('%s.%s.%s', v_schema_name, v_table_name, v_op::text), - pgconductor._private_build_column_list(sub.column_names, 'old'), - pgconductor._private_build_column_list(sub.column_names, 'new'), - sub.id - ), e'\n') - from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue - where sub.table_name = v_table_name - and sub.schema_name = v_schema_name - and sub.operation = v_op - ); - - -- Create trigger function - execute format( - $sql$ - create or replace function pgconductor._private_trigger_event_%s_on_%I_%I() - returns trigger - language plpgsql - security definer - set search_path to '' - as $inner$ - declare - v_task_keys text[]; - v_queues text[]; - v_payloads jsonb[]; - v_subscription_ids uuid[]; - begin - %s - - if array_length(v_task_keys, 1) > 0 then - insert into pgconductor._private_executions (task_key, queue, payload, subscription_id) - select unnest(v_task_keys), unnest(v_queues), unnest(v_payloads), unnest(v_subscription_ids); - end if; - - if tg_op = 'DELETE' then - return old; - end if; - - return new; - end - $inner$ - $sql$, - v_op::text, - v_schema_name, - v_table_name, - v_invoke_blocks - ); - - -- Create trigger - execute format( - $sql$ - create trigger pgconductor_event_%s - after %s on %I.%I - for each row - execute function pgconductor._private_trigger_event_%s_on_%I_%I() - $sql$, - v_op::text, - upper(v_op::text), - v_schema_name, - v_table_name, - v_op::text, - v_schema_name, - v_table_name - ); - end if; - end loop; - - if tg_op = 'DELETE' then - return old; - end if; - - return new; -end; -$_$; - -create trigger sync_database_trigger - after insert or delete or update on pgconductor._private_event_subscriptions - for each row - execute function pgconductor._private_sync_database_trigger(); - -create or replace function pgconductor.emit_event( - p_event_key text, - p_payload jsonb default '{}'::jsonb -) - returns uuid - language sql - volatile - set search_path to '' -as $_$ - insert into pgconductor._private_custom_events (event_key, payload) - values (p_event_key, p_payload) - returning id; -$_$; diff --git a/packages/pgconductor-js/src/conductor.ts b/packages/pgconductor-js/src/conductor.ts index 52116c6..0d248d8 100644 --- a/packages/pgconductor-js/src/conductor.ts +++ b/packages/pgconductor-js/src/conductor.ts @@ -20,24 +20,23 @@ import { type TaskName, type Trigger, type ValidateTriggers, + type ValidateEventTriggers, } from "./task-definition"; -import { Worker, type WorkerConfig } from "./worker"; +import { EVENT_DISPATCH_QUEUE, Worker, type WorkerConfig } from "./worker"; import { DefaultLogger, type Logger } from "./lib/logger"; import { SchemaManager } from "./schema-manager"; import type { EventDefinition, - GenericDatabase, EventName, FindEventByIdentifier, InferEventPayload, } from "./event-definition"; +import { validateEventTriggers } from "./event-trigger-validation"; import { TaskSchemas, EventSchemas, - DatabaseSchema, type InferTasksFromSchema, type InferEventsFromSchema, - type InferDatabaseFromSchema, } from "./schemas"; type ConnectionOptions = @@ -67,15 +66,12 @@ type ResolvedReturns< export type ConductorOptions< TTaskSchemas extends TaskSchemas | undefined, TEventSchemas extends EventSchemas | undefined, - TDatabaseSchema extends DatabaseSchema | undefined, ExtraContext extends object, > = ConnectionOptions & { tasks?: TTaskSchemas; events?: TEventSchemas; - database?: TDatabaseSchema; - context: ExtraContext; logger?: Logger; @@ -86,13 +82,12 @@ export type ConductorOptions< export class Conductor< TTaskSchemas extends TaskSchemas | undefined = undefined, TEventSchemas extends EventSchemas | undefined = undefined, - TDatabaseSchema extends DatabaseSchema | undefined = undefined, ExtraContext extends object = {}, // Inferred types from schemas Tasks extends readonly TaskDefinition[] = InferTasksFromSchema, - Events extends readonly EventDefinition[] = InferEventsFromSchema, - Database extends GenericDatabase = InferDatabaseFromSchema, + Events extends readonly EventDefinition[] = + InferEventsFromSchema, > { /** * @internal @@ -107,12 +102,7 @@ export class Conductor< readonly logger: Logger; private constructor( - public readonly options: ConductorOptions< - TTaskSchemas, - TEventSchemas, - TDatabaseSchema, - ExtraContext - >, + public readonly options: ConductorOptions, ) { this.logger = options.logger || new DefaultLogger(); @@ -131,18 +121,16 @@ export class Conductor< static create< TTaskSchemas extends TaskSchemas | undefined = undefined, TEventSchemas extends EventSchemas | undefined = undefined, - TDatabaseSchema extends DatabaseSchema | undefined = undefined, TExtraContext extends object = {}, >( options: ConnectionOptions & { tasks?: TTaskSchemas; events?: TEventSchemas; - database?: TDatabaseSchema; context: TExtraContext; logger?: Logger; }, - ): Conductor { - return new Conductor(options); + ): Conductor { + return new Conductor(options); } /** @@ -164,23 +152,21 @@ export class Conductor< const TTriggers extends object | readonly object[], >( definition: TDef & ValidateDeadLetterConfiguration>, - triggers: TTriggers & ValidateTriggers>, + triggers: TTriggers & + ValidateTriggers> & + ValidateEventTriggers, fn: TDef extends { readonly batch: BatchConfig } ? ResolvedReturns extends void ? ( - events: Array< - TaskEventFromTriggers, Events, Database> - >, + events: Array, Events>>, ctx: BatchTaskContext, ) => Promise : ( - events: Array< - TaskEventFromTriggers, Events, Database> - >, + events: Array, Events>>, ctx: BatchTaskContext, ) => Promise>> : ( - event: TaskEventFromTriggers, Events, Database>, + event: TaskEventFromTriggers, Events>, ctx: TaskContext & ExtraContext, ) => Promise>, ): Task< @@ -189,15 +175,20 @@ export class Conductor< ResolvedPayload, ResolvedReturns, TaskContext & ExtraContext, - TaskEventFromTriggers, Events, Database> + TaskEventFromTriggers, Events> > { + validateEventTriggers( + triggers, + this.options.events?.definitions ?? [], + this.options.events?.hasTypeOnlyDefinitions ?? false, + ); return Task.create< TDef["name"], ResolvedQueue, ResolvedPayload, ResolvedReturns, TaskContext & ExtraContext, - TaskEventFromTriggers, Events, Database> + TaskEventFromTriggers, Events> >( definition as TaskConfiguration< TDef["name"], @@ -206,7 +197,7 @@ export class Conductor< >, triggers as NonEmptyArray | Trigger, fn as ExecuteFunction< - TaskEventFromTriggers, Events, Database>, + TaskEventFromTriggers, Events>, ResolvedReturns, TaskContext & ExtraContext >, @@ -221,6 +212,9 @@ export class Conductor< tasks: ValidateTasksQueue; config?: Partial; }): Worker { + if (options.queue === EVENT_DISPATCH_QUEUE) { + throw new Error(`Queue "${EVENT_DISPATCH_QUEUE}" is reserved for internal use`); + } return new Worker( options.queue, options.tasks as AnyTask[], @@ -228,6 +222,9 @@ export class Conductor< this.logger, options.config, this.options.context, + this.options.events?.definitions ?? [], + true, + this.options.events?.hasTypeOnlyDefinitions ?? false, ); } @@ -298,6 +295,20 @@ export class Conductor< TName extends EventName, TDef extends FindEventByIdentifier = FindEventByIdentifier, >(event: TName, payload: InferEventPayload): Promise { + // Runtime schemas are deliberately validated before the database call. This + // keeps emit a persistence boundary: an invalid event can never be queued. + const definition = this.options.events?.definitions.find( + (candidate: EventDefinition) => candidate.name === event, + ); + const standard = (definition?.payload as any)?.["~standard"]; + if (standard?.validate) { + const result = await standard.validate(payload); + if (result && typeof result === "object" && "issues" in result && result.issues) { + throw new Error(`Invalid payload for event "${String(event)}"`); + } + payload = ((result as any)?.value ?? payload) as InferEventPayload; + } + return this.db.emitEvent({ eventKey: event, payload: payload as any, diff --git a/packages/pgconductor-js/src/database-client.ts b/packages/pgconductor-js/src/database-client.ts index 80d4f5f..be15d18 100644 --- a/packages/pgconductor-js/src/database-client.ts +++ b/packages/pgconductor-js/src/database-client.ts @@ -10,6 +10,8 @@ import { type CountActiveOrchestratorsBelowArgs, type GetExecutionsArgs, type RemoveExecutionsArgs, + type RemoveCustomEventsArgs, + type DispatchCustomEventsArgs, type RegisterWorkerArgs, type ScheduleCronExecutionArgs, type UnscheduleCronExecutionArgs, @@ -74,6 +76,8 @@ export interface Execution { dedupe_key?: string | null; cron_expression?: string | null; group?: string | null; + source_event_id?: string | null; + event_subscription_id?: string | null; dead_letter_source_execution_id?: string | null; dead_letter_source_queue?: string | null; dead_letter_source_task_key?: string | null; @@ -154,14 +158,9 @@ export interface ExecutionInvokeChild { export interface EventSubscriptionSpec { task_key: string; - queue: string; - event_key: string | null; - schema_name: string | null; - table_name: string | null; - operation: "insert" | "update" | "delete" | null; - when_clause: string | null; + event_key: string; payload_fields: string[] | null; - column_names: string[] | null; + filter: Record | null; } const RETRYABLE_SQLSTATE_CODES = new Set([ @@ -508,6 +507,34 @@ export class DatabaseClient { return deletedCount >= args.batchSize; } + async removeCustomEvents( + before: Date, + batchSize: number, + opts?: QueryMethodOptions, + ): Promise; + async removeCustomEvents( + args: RemoveCustomEventsArgs, + opts?: QueryMethodOptions, + ): Promise; + async removeCustomEvents( + beforeOrArgs: Date | RemoveCustomEventsArgs, + batchSizeOrOpts?: number | QueryMethodOptions, + opts?: QueryMethodOptions, + ): Promise { + const args: RemoveCustomEventsArgs = + beforeOrArgs instanceof Date + ? { before: beforeOrArgs, batchSize: batchSizeOrOpts as number } + : beforeOrArgs; + const options = beforeOrArgs instanceof Date ? opts : (batchSizeOrOpts as QueryMethodOptions); + const result = await this.query(() => this.builder.buildRemoveCustomEvents(args), { + label: "removeCustomEvents", + ...options, + }); + + const deletedCount = result[0]?.deleted_count ?? 0; + return deletedCount >= args.batchSize; + } + async registerWorker(args: RegisterWorkerArgs, opts?: QueryMethodOptions): Promise { await this.query(() => this.builder.buildRegisterWorker(args), { label: "registerWorker", @@ -581,6 +608,17 @@ export class DatabaseClient { }); } + async dispatchCustomEvents( + args: DispatchCustomEventsArgs, + opts?: QueryMethodOptions, + ): Promise { + const rows = await this.query(() => this.builder.buildDispatchCustomEvents(args), { + label: "dispatchCustomEvents", + ...opts, + }); + return rows.map((row) => row.event_id); + } + async emitEvent(args: EmitEventArgs, opts?: QueryMethodOptions): Promise { const result = await this.query(() => this.builder.buildEmitEvent(args), { label: "emitEvent", diff --git a/packages/pgconductor-js/src/event-definition.ts b/packages/pgconductor-js/src/event-definition.ts index 6825092..2c75dd1 100644 --- a/packages/pgconductor-js/src/event-definition.ts +++ b/packages/pgconductor-js/src/event-definition.ts @@ -1,11 +1,43 @@ import type { StandardSchemaV1 } from "@standard-schema/spec"; -import type { ColumnSelectionError, SelectedRow, ValidateColumns } from "./select-columns"; type ObjectSchema = StandardSchemaV1; +type EnsureObject = T extends object ? T : {}; +type SchemaOutput = T extends StandardSchemaV1 ? O : T; +type EventPayload = T extends undefined ? {} : EnsureObject>; + +export type EventFilterScalar = string | number | boolean | null; +type ScalarFilterValue = Extract; -export type EventDefinition = { +export type FilterableKeys = { + [K in keyof EventPayload & string]: Exclude< + EventPayload[K], + undefined + > extends EventFilterScalar + ? Exclude[K], undefined> extends never + ? never + : K + : never; +}[keyof EventPayload & string]; + +export type EventFilter = { + readonly [K in FilterableKeys]?: readonly ScalarFilterValue[K]>[]; +}; + +export type FilterForEvent = + T extends EventDefinition + ? { + readonly [F in K & FilterableKeys

]?: readonly ScalarFilterValue[F]>[]; + } + : never; + +export type EventDefinition< + Name extends string, + Payload = undefined, + Filterable extends string = never, +> = { readonly name: Name; readonly payload: Payload; + readonly filterable?: readonly Filterable[]; }; /** @@ -21,75 +53,43 @@ export type DefineEvent< T extends { name: string; payload?: unknown; + filterable?: readonly FilterableKeys[]; }, -> = EventDefinition; +> = EventDefinition< + T["name"], + T extends { payload: infer P } ? P : undefined, + T extends { filterable: readonly (infer K extends string)[] } ? K : never +>; export function defineEvent(def: { name: Name; payload?: Payload; + filterable?: undefined; }): EventDefinition; +export function defineEvent< + Name extends string, + Payload extends ObjectSchema, + const Filterable extends readonly FilterableKeys[], +>(def: { + name: Name; + payload: Payload; + filterable: Filterable; +}): EventDefinition; export function defineEvent(def: any) { return def; } -export type EventName[]> = +export type EventName[]> = TEvents[number]["name"]; export type FindEventByIdentifier< - TEvents extends readonly EventDefinition[], + TEvents extends readonly EventDefinition[], TName extends string, > = Extract; -type EnsureObject = T extends object ? T : {}; - export type InferEventPayload = - T extends EventDefinition - ? P extends undefined - ? {} - : P extends StandardSchemaV1 - ? EnsureObject - : EnsureObject

// Plain type (type-only definition) - : never; - -export type GenericDatabase = Record>; -export type SchemaName = keyof TDatabase; -export type TableName< - TDatabase extends GenericDatabase, - TSchema extends SchemaName, -> = keyof TDatabase[TSchema]; -export type RowType< - TDatabase extends GenericDatabase, - TSchema extends SchemaName, - TTable extends TableName, -> = TDatabase[TSchema][TTable]; -export type DatabaseEventPayload< - TRow, - TOp extends "insert" | "update" | "delete", - TSelection extends string, -> = - SelectedRow extends infer Selection - ? Selection extends ColumnSelectionError - ? Selection - : { - old: TOp extends "delete" | "update" ? Selection : null; - new: TOp extends "insert" | "update" ? Selection : null; - tg_table: string; - tg_op: Uppercase; - } - : never; + T extends EventDefinition ? EventPayload

: never; export type CustomEventConfig = { event: TName; }; -export type DatabaseEventConfig< - TDatabase extends GenericDatabase, - TSchema extends SchemaName, - TTable extends TableName, - TOp extends "insert" | "update" | "delete", - TSelection extends string = string, -> = { - schema: TSchema; - table: TTable; - operation: TOp; - columns: ValidateColumns>; -}; diff --git a/packages/pgconductor-js/src/event-trigger-validation.ts b/packages/pgconductor-js/src/event-trigger-validation.ts new file mode 100644 index 0000000..2f29f08 --- /dev/null +++ b/packages/pgconductor-js/src/event-trigger-validation.ts @@ -0,0 +1,184 @@ +import type { EventDefinition } from "./event-definition"; +import type { EventSubscriptionSpec, JsonValue } from "./database-client"; + +const EVENT_FIELD_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const MAX_EVENT_NAME_BYTES = 255; +const MAX_EVENT_FIELD_BYTES = 128; +const MAX_FILTER_VALUE_BYTES = 1024; +const MAX_FILTER_FIELDS = 8; +const MAX_FILTER_ALTERNATIVES = 4; + +function utf8ByteLength(value: string): number { + return new TextEncoder().encode(value).length; +} + +function assertFieldName(field: string, eventName: string): void { + if (!EVENT_FIELD_PATTERN.test(field)) { + throw new Error(`Fields for event "${eventName}" contains invalid field "${field}"`); + } + if (utf8ByteLength(field) > MAX_EVENT_FIELD_BYTES) { + throw new Error(`Field "${field}" for event "${eventName}" exceeds 128 UTF-8 bytes`); + } +} + +export function parseEventPayloadFields(fields: unknown, eventName: string): string[] | null { + if (fields === undefined) return null; + if (typeof fields !== "string") { + throw new Error(`Fields for event "${eventName}" must be a comma-separated string`); + } + + const selected = fields.split(",").map((field) => field.trim()); + if (selected.some((field) => field.length === 0)) { + throw new Error(`Fields for event "${eventName}" cannot contain empty names`); + } + for (const field of selected) { + assertFieldName(field, eventName); + } + if (new Set(selected).size !== selected.length) { + throw new Error(`Fields for event "${eventName}" cannot contain duplicate names`); + } + return selected; +} + +function isEventFilterScalar(value: unknown): value is string | number | boolean | null { + return ( + value === null || + typeof value === "string" || + typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)) + ); +} + +function scalarSortKey(value: string | number | boolean | null): string { + if (value === null) return "0:null"; + if (typeof value === "boolean") return `1:${value ? "true" : "false"}`; + if (typeof value === "number") return `2:${JSON.stringify(value)}`; + return `3:${JSON.stringify(value)}`; +} + +function scalarByteLength(value: string | number | boolean | null): number { + return utf8ByteLength(JSON.stringify(value)); +} + +function canonicalFilter( + filter: unknown, + eventName: string, + definition: EventDefinition | undefined, + allowUnknownEvents: boolean, +): Record | null { + if (filter === undefined || filter === null) return null; + if (typeof filter !== "object" || Array.isArray(filter)) { + throw new Error(`Filter for event "${eventName}" must be an object`); + } + + const filterEntries = Object.entries(filter as Record); + if (filterEntries.length > MAX_FILTER_FIELDS) { + throw new Error(`Filter for event "${eventName}" supports at most 8 fields`); + } + if (!definition && !allowUnknownEvents) { + throw new Error(`Filtered event "${eventName}" has no runtime event definition`); + } + + const allowed = definition ? new Set(definition.filterable ?? []) : undefined; + const canonical: Record = {}; + for (const [field, values] of filterEntries.sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + )) { + if (utf8ByteLength(field) === 0) { + throw new Error(`Filter for event "${eventName}" cannot contain empty field names`); + } + if (utf8ByteLength(field) > MAX_EVENT_FIELD_BYTES) { + throw new Error(`Filter field "${field}" for event "${eventName}" exceeds 128 UTF-8 bytes`); + } + if (allowed && !allowed.has(field)) { + throw new Error(`Filter for event "${eventName}" contains undeclared field "${field}"`); + } + if (!Array.isArray(values)) { + throw new Error(`Filter value for event "${eventName}" field "${field}" must be an array`); + } + if (values.length === 0) { + throw new Error(`Filter value for event "${eventName}" field "${field}" cannot be empty`); + } + if (values.length > MAX_FILTER_ALTERNATIVES) { + throw new Error( + `Filter value for event "${eventName}" field "${field}" supports at most 4 values`, + ); + } + + const byKey = new Map(); + for (const value of values) { + if (!isEventFilterScalar(value)) { + throw new Error( + `Filter value for event "${eventName}" field "${field}" must contain only scalar values`, + ); + } + if (scalarByteLength(value) > MAX_FILTER_VALUE_BYTES) { + throw new Error( + `Filter value for event "${eventName}" field "${field}" exceeds 1024 UTF-8 bytes`, + ); + } + byKey.set(scalarSortKey(value), value); + } + canonical[field] = [...byKey.entries()] + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([, value]) => value); + } + return canonical; +} + +export type CompiledEventTrigger = Pick< + EventSubscriptionSpec, + "event_key" | "payload_fields" | "filter" +>; + +/** Validate and canonicalize one trigger. Non-event triggers return null. */ +export function compileEventTrigger( + trigger: object, + eventDefinitions: readonly EventDefinition[], + allowUnknownEvents = false, +): CompiledEventTrigger | null { + if (!("event" in trigger)) return null; + const candidate = trigger as Record; + if (typeof candidate.event !== "string" || candidate.event.trim().length === 0) { + throw new Error("Custom event triggers require a non-empty event name"); + } + + const eventName = candidate.event; + if (utf8ByteLength(eventName) > MAX_EVENT_NAME_BYTES) { + throw new Error(`Event "${eventName}" exceeds 255 UTF-8 bytes`); + } + if ("when" in candidate) { + throw new Error(`Custom event "${eventName}" does not support a when clause`); + } + + const definition = eventDefinitions.find((event) => event.name === eventName); + if (eventDefinitions.length > 0 && !definition && !allowUnknownEvents) { + throw new Error(`Event "${eventName}" is not defined in the conductor event catalog`); + } + + return { + event_key: eventName, + payload_fields: parseEventPayloadFields(candidate.fields, eventName), + filter: canonicalFilter(candidate.filter, eventName, definition, allowUnknownEvents), + }; +} + +export function compileEventTriggers( + triggers: object | readonly object[], + eventDefinitions: readonly EventDefinition[], + allowUnknownEvents = false, +): CompiledEventTrigger[] { + const list = Array.isArray(triggers) ? triggers : [triggers]; + return list.flatMap((trigger) => { + const compiled = compileEventTrigger(trigger, eventDefinitions, allowUnknownEvents); + return compiled ? [compiled] : []; + }); +} + +export function validateEventTriggers( + triggers: object | readonly object[], + eventDefinitions: readonly EventDefinition[], + allowUnknownEvents = false, +): void { + compileEventTriggers(triggers, eventDefinitions, allowUnknownEvents); +} diff --git a/packages/pgconductor-js/src/generated/sql.ts b/packages/pgconductor-js/src/generated/sql.ts index 199045a..7912a4e 100644 --- a/packages/pgconductor-js/src/generated/sql.ts +++ b/packages/pgconductor-js/src/generated/sql.ts @@ -294,7 +294,11 @@ begin RETURN NEW; elsif tg_op = 'UPDATE' then - -- protect default queue from modification + -- protect queues required by the runtime + if old.name = 'pgconductor.internal' or new.name = 'pgconductor.internal' then + raise exception 'Modifying the internal queue is not allowed'; + end if; + if old.name = 'default' or new.name = 'default' then raise exception 'Modifying the default queue is not allowed'; end if; @@ -307,7 +311,11 @@ begin return new; elsif tg_op = 'DELETE' then - -- protect default queue from deletion + -- protect queues required by the runtime + if old.name = 'pgconductor.internal' then + raise exception 'Deleting the internal queue is not allowed'; + end if; + if old.name = 'default' then raise exception 'Deleting the default queue is not allowed'; end if; @@ -371,22 +379,11 @@ create type pgconductor.task_spec as ( dead_letter_task_key text ); -create type pgconductor._private_event_operation as enum ( - 'insert', - 'update', - 'delete' -); - create type pgconductor.event_subscription_spec as ( task_key text, - queue text, event_key text, - schema_name text, - table_name text, - operation pgconductor._private_event_operation, - when_clause text, payload_fields text[], - column_names text[] + filter jsonb ); create or replace function pgconductor._private_register_worker( @@ -401,19 +398,35 @@ volatile set search_path to '' as $function$ begin - -- step 1: upsert queue (triggers partition creation) + -- Upsert every required queue in a stable order (triggers partition + -- creation). Registrations may reference one another as dead-letter queues, + -- so queue creation and locking share a global order. insert into pgconductor._private_queues (name) - values (p_queue_name) + select required.name + from ( + select p_queue_name as name + union + select spec.dead_letter_queue + from unnest(p_task_specs) as spec + where spec.dead_letter_queue is not null + ) required + order by required.name on conflict (name) do nothing; - -- Dead-letter destinations may not have a worker yet; create their partitions. - insert into pgconductor._private_queues (name) - select distinct spec.dead_letter_queue - from unnest(p_task_specs) as spec - where spec.dead_letter_queue is not null - on conflict (name) do nothing; + -- Queue row locks are the only registration locks; acquire them in order. + perform 1 + from pgconductor._private_queues queue + where queue.name in ( + select p_queue_name + union + select spec.dead_letter_queue + from unnest(p_task_specs) as spec + where spec.dead_letter_queue is not null + ) + order by queue.name + for update; - -- step 2: register/update tasks + -- Register/update tasks. insert into pgconductor._private_tasks (key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days, window_start, window_end, concurrency_limit, group_concurrency_limit, dead_letter_queue, dead_letter_task_key) select spec.key, @@ -441,7 +454,7 @@ begin dead_letter_queue = excluded.dead_letter_queue, dead_letter_task_key = excluded.dead_letter_task_key; - -- step 3: insert scheduled cron executions + -- Insert scheduled cron executions. insert into pgconductor._private_executions (task_key, queue, payload, run_at, dedupe_key, cron_expression, "group") select spec.task_key, @@ -459,8 +472,7 @@ begin cron_expression = excluded.cron_expression, "group" = excluded."group"; - -- step 4: clean up stale schedules for this queue - -- delete future executions for schedules that no longer exist + -- Clean up stale schedules for this queue. delete from pgconductor._private_executions where queue = p_queue_name and cron_expression is not null @@ -472,7 +484,6 @@ begin where spec.dedupe_key is not null and spec.dedupe_key like 'scheduled::%' ); - -- mark running executions as cancelled for schedules that no longer exist update pgconductor._private_executions set cancelled = true where queue = p_queue_name @@ -488,59 +499,10 @@ begin where spec.dedupe_key is not null and spec.dedupe_key like 'scheduled::%' ); - -- step 5: manage event subscriptions (only recreate triggers when subscriptions change) - merge into pgconductor._private_event_subscriptions as target - using ( - select - s.task_key, - s.queue, - s.event_key, - s.schema_name, - s.table_name, - s.operation, - s.when_clause, - s.payload_fields, - s.column_names - from unnest(p_event_subscriptions) as s - ) as source - on ( - target.queue = source.queue and - target.task_key = source.task_key and - coalesce(target.event_key, '') = coalesce(source.event_key, '') and - coalesce(target.schema_name, '') = coalesce(source.schema_name, '') and - coalesce(target.table_name, '') = coalesce(source.table_name, '') and - coalesce(target.operation::text, '') = coalesce(source.operation::text, '') and - coalesce(target.when_clause, '') = coalesce(source.when_clause, '') and - coalesce(array_to_string(target.payload_fields, ','), '') = - coalesce(array_to_string(source.payload_fields, ','), '') and - coalesce(array_to_string(target.column_names, ','), '') = - coalesce(array_to_string(source.column_names, ','), '') - ) - when not matched then insert ( - task_key, queue, event_key, schema_name, table_name, operation, - when_clause, payload_fields, column_names - ) values ( - source.task_key, source.queue, source.event_key, - source.schema_name, source.table_name, source.operation, - source.when_clause, source.payload_fields, source.column_names + perform pgconductor._private_replace_custom_event_subscriptions( + p_queue_name, + p_event_subscriptions ); - - -- step 6: delete old subscriptions for this queue not in new set - delete from pgconductor._private_event_subscriptions target - where target.queue = p_queue_name - and not exists ( - select 1 from unnest(p_event_subscriptions) source - where target.task_key = source.task_key - and coalesce(target.event_key, '') = coalesce(source.event_key, '') - and coalesce(target.schema_name, '') = coalesce(source.schema_name, '') - and coalesce(target.table_name, '') = coalesce(source.table_name, '') - and coalesce(target.operation::text, '') = coalesce(source.operation::text, '') - and coalesce(target.when_clause, '') = coalesce(source.when_clause, '') - and coalesce(array_to_string(target.payload_fields, ','), '') = - coalesce(array_to_string(source.payload_fields, ','), '') - and coalesce(array_to_string(target.column_names, ','), '') = - coalesce(array_to_string(source.column_names, ','), '') - ); end; $function$; @@ -924,353 +886,414 @@ begin end; $function$; -`, - "0000000002_events.sql": String.raw` +-- Event deliveries are independent executions. The source execution has the +-- same UUID as its event-log row; destinations carry the event identity and +-- subscription identity in dedicated columns rather than using workflow +-- parentage. alter table pgconductor._private_executions - add column if not exists subscription_id uuid; - -create table if not exists pgconductor._private_custom_events ( - id uuid default pgconductor._private_portable_uuidv7() not null, - event_key text not null, - payload jsonb not null default '{}'::jsonb, - created_at timestamptz default pgconductor._private_current_time() not null, - primary key (created_at, id) -) partition by range (created_at); + add column source_event_id uuid, + add column event_subscription_id uuid, + drop column if exists subscription_id; -create index if not exists idx_custom_events_event_key - on pgconductor._private_custom_events (event_key, created_at desc); +alter table pgconductor._private_executions + add constraint chk_executions_event_delivery_pair + check ((source_event_id is null) = (event_subscription_id is null)); --- Create initial partition for custom events (will cover many years) -create table if not exists pgconductor._private_custom_events_default - partition of pgconductor._private_custom_events - for values from (minvalue) to (maxvalue); +alter table pgconductor._private_executions + add constraint chk_executions_event_delivery_no_parent + check (source_event_id is null or parent_execution_id is null); + +create unique index idx_executions_event_destination + on pgconductor._private_executions (source_event_id, event_subscription_id, queue) + where source_event_id is not null and event_subscription_id is not null; + +-- A small immutable helper is used by the generated subscription field count. +create or replace function pgconductor._private_jsonb_object_size(p_value jsonb) +returns integer +language sql +immutable +strict +set search_path to '' +as $function$ + select count(*)::integer from jsonb_object_keys(p_value); +$function$; -create table if not exists pgconductor._private_event_subscriptions ( +create table pgconductor._private_custom_event_subscriptions ( id uuid primary key default pgconductor._private_portable_uuidv7(), - + event_key text not null, task_key text not null, queue text not null, - - event_key text, - schema_name text, - table_name text, - operation pgconductor._private_event_operation, - - when_clause text, payload_fields text[], - column_names text[], - + filter jsonb not null default '{}'::jsonb, + field_count integer generated always as + (pgconductor._private_jsonb_object_size(filter)) stored, created_at timestamptz not null default pgconductor._private_current_time(), - - constraint chk_event_type check ( - (event_key is not null and schema_name is null and table_name is null and operation is null) - or - (event_key is null and schema_name is not null and table_name is not null and operation is not null) + constraint chk_custom_event_subscription_event_key check ( + btrim(event_key) <> '' and octet_length(event_key) between 1 and 255 + ), + constraint chk_custom_event_subscription_filter_object check ( + jsonb_typeof(filter) = 'object' ) ); -create index if not exists idx_event_subscriptions_custom - on pgconductor._private_event_subscriptions (event_key) - where event_key is not null; +create index idx_custom_event_subscriptions_event + on pgconductor._private_custom_event_subscriptions (event_key, id); +create index idx_custom_event_subscriptions_unfiltered + on pgconductor._private_custom_event_subscriptions (event_key, id) + where field_count = 0; -create index if not exists idx_event_subscriptions_database - on pgconductor._private_event_subscriptions (schema_name, table_name, operation) - where schema_name is not null; - -create or replace function pgconductor._private_sync_custom_event_trigger() - returns trigger - language plpgsql - security definer - set search_path to '' -as $_$ -declare - v_invoke_blocks text; - v_has_subscriptions boolean; -begin - -- Only process custom event subscriptions (event_key is not null) - if coalesce(new.event_key, old.event_key) is null then - return coalesce(new, old); - end if; - - drop trigger if exists pgconductor_custom_event on pgconductor._private_custom_events; - drop function if exists pgconductor._private_trigger_custom_event; - - select exists( - select 1 - from pgconductor._private_event_subscriptions - where event_key is not null - ) into v_has_subscriptions; - - if v_has_subscriptions then - v_invoke_blocks := ( - select string_agg(format( - $sql$ - if new.event_key = %L and (%s) then - v_task_keys := array_append(v_task_keys, %L); - v_queues := array_append(v_queues, %L); - v_payloads := array_append(v_payloads, jsonb_build_object('event', new.event_key, 'payload', %s)); - v_subscription_ids := array_append(v_subscription_ids, %L); - end if; - $sql$, - sub.event_key, - coalesce(nullif(sub.when_clause, ''), 'true'), - sub.task_key, - t.queue, - pgconductor._private_build_payload_fields(sub.payload_fields, 'new.payload'), - sub.id - ), e'\n') - from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue - where sub.event_key is not null - ); - - execute format( - $sql$ - create or replace function pgconductor._private_trigger_custom_event() - returns trigger - language plpgsql - security definer - set search_path to '' - as $inner$ - declare - v_task_keys text[]; - v_queues text[]; - v_payloads jsonb[]; - v_subscription_ids uuid[]; - begin - %s - - if array_length(v_task_keys, 1) > 0 then - insert into pgconductor._private_executions (task_key, queue, payload, subscription_id) - select unnest(v_task_keys), unnest(v_queues), unnest(v_payloads), unnest(v_subscription_ids); - end if; - - return new; - end - $inner$ - $sql$, - v_invoke_blocks - ); - - execute $sql$ - create trigger pgconductor_custom_event - after insert on pgconductor._private_custom_events - for each row - execute function pgconductor._private_trigger_custom_event() - $sql$; - end if; +create table pgconductor._private_custom_event_predicates ( + id bigint generated always as identity primary key, + subscription_id uuid not null references pgconductor._private_custom_event_subscriptions(id) on delete cascade, + event_key text not null, + field_name text not null, + value jsonb not null, + constraint uq_custom_event_predicate_alternative + unique (subscription_id, field_name, value), + constraint chk_custom_event_predicate_event_key check ( + btrim(event_key) <> '' and octet_length(event_key) between 1 and 255 + ), + constraint chk_custom_event_predicate_field_name check ( + btrim(field_name) <> '' and octet_length(field_name) <= 128 + ), + constraint chk_custom_event_predicate_scalar check ( + jsonb_typeof(value) in ('string', 'number', 'boolean', 'null') + ), + constraint chk_custom_event_predicate_scalar_text_size check ( + octet_length(value::text) <= 1024 + ) +); - if tg_op = 'DELETE' then - return old; - end if; +create index idx_custom_event_predicates_exact + on pgconductor._private_custom_event_predicates + (event_key, field_name, value, subscription_id); - return new; -end; -$_$; +-- Persistent worker subscriptions are replaced as one queue-scoped snapshot. +-- Canonicalization and policy validation happen in TypeScript; database checks +-- retain only storage and index integrity guarantees. +create or replace function pgconductor._private_replace_custom_event_subscriptions( + p_queue_name text, + p_subscriptions pgconductor.event_subscription_spec[] +) +returns void +language sql +volatile +set search_path to '' +as $function$ + with removed as ( + delete from pgconductor._private_custom_event_subscriptions + where queue = p_queue_name + ), input as materialized ( + select task_key, event_key, payload_fields, filter, input_ordinal + from unnest(p_subscriptions) with ordinality + as subscription(task_key, event_key, payload_fields, filter, input_ordinal) + ), inserted as ( + insert into pgconductor._private_custom_event_subscriptions ( + id, event_key, task_key, queue, payload_fields, filter + ) + select + pgconductor._private_portable_uuidv7(), + input.event_key, + input.task_key, + p_queue_name, + input.payload_fields, + coalesce(input.filter, '{}'::jsonb) + from input + order by input.input_ordinal + returning id, event_key, filter + ) + insert into pgconductor._private_custom_event_predicates ( + subscription_id, event_key, field_name, value + ) + select + inserted.id, + inserted.event_key, + field.key, + alternative.value + from inserted + cross join lateral jsonb_each(inserted.filter) as field + cross join lateral jsonb_array_elements(field.value) as alternative(value); +$function$; -create trigger sync_custom_event_trigger - after insert or delete or update on pgconductor._private_event_subscriptions - for each row - execute function pgconductor._private_sync_custom_event_trigger(); +create table pgconductor._private_custom_events ( + id uuid primary key, + event_position bigint generated always as identity unique, + event_key text not null, + payload jsonb not null, + created_at timestamptz not null default pgconductor._private_current_time(), + dispatched_at timestamptz, + constraint chk_custom_event_event_key check ( + btrim(event_key) <> '' and octet_length(event_key) between 1 and 255 + ), + constraint chk_custom_event_payload_object check ( + jsonb_typeof(payload) = 'object' + ) +); -create or replace function pgconductor._private_build_payload_fields( - p_payload_fields text[], - p_payload_expr text +-- event_position is a deterministic replay cursor, not commit order. A future +-- one-shot wait implementation must define its own no-miss registration boundary. +create index idx_custom_events_replay + on pgconductor._private_custom_events (event_key, event_position); +create index idx_custom_events_retention + on pgconductor._private_custom_events (dispatched_at, event_position) + where dispatched_at is not null; +create index idx_custom_events_terminal_cleanup + on pgconductor._private_custom_events (created_at, event_position); + +insert into pgconductor._private_queues (name) +values ('pgconductor.internal') +on conflict do nothing; + +insert into pgconductor._private_tasks ( + key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days +) +values ( + 'pgconductor.event-dispatch', 'pgconductor.internal', 3, 0, null ) - returns text - language sql - immutable - set search_path to '' -as $_$ +on conflict (queue, key) do update set + max_attempts = excluded.max_attempts, + remove_on_complete_days = excluded.remove_on_complete_days, + remove_on_fail_days = excluded.remove_on_fail_days; + +create or replace function pgconductor._private_extract_event_payload(p_fields text[], p_payload jsonb) +returns jsonb +language sql +immutable +set search_path to '' +as $function$ select case - when p_payload_fields is null then p_payload_expr - else 'jsonb_build_object(' || array_to_string( - array( - select format('%L, %s->%L', field, p_payload_expr, field) - from unnest(p_payload_fields) as field + when p_fields is null then p_payload + else coalesce( + ( + select jsonb_object_agg(key, p_payload -> key) + from unnest(p_fields) key + where p_payload ? key ), - ', ' - ) || ')' + '{}'::jsonb + ) end; -$_$; +$function$; -create or replace function pgconductor._private_build_column_list( - p_column_names text[], - p_record_name text +-- Claiming, matching, destination insertion, and marking the source dispatched +-- are one statement. The write barrier makes the UPDATE depend on the +-- destination INSERT even when an event has zero matches. +create or replace function pgconductor._private_dispatch_custom_events( + p_event_ids uuid[], + p_orchestrator_id uuid ) - returns text - language sql - immutable - set search_path to '' -as $_$ - select case - when p_column_names is null then format('row_to_json(%I.*)', p_record_name) - else 'jsonb_build_object(' || array_to_string( - array( - select format('%L, %I.%I', col, p_record_name, col) - from unnest(p_column_names) as col +returns table(event_id uuid) +language sql +volatile +security definer +set search_path to '' +as $function$ + with claimed as materialized ( + select + source.id, + source_event.event_key, + source_event.payload, + source_event.dispatched_at + from pgconductor._private_executions source + join pgconductor._private_custom_events source_event + on source_event.id = source.id + where source.id = any(coalesce(p_event_ids, array[]::uuid[])) + and source.queue = 'pgconductor.internal' + and source.task_key = 'pgconductor.event-dispatch' + and source.locked_by = p_orchestrator_id + and source.completed_at is null + and source.failed_at is null + and not source.cancelled + for update of source, source_event + ), pending as materialized ( + select id, event_key, payload + from claimed + where dispatched_at is null + ), event_values as materialized ( + select + pending.id as event_id, + pending.event_key, + pending.payload, + field.key as field_name, + field.value + from pending + cross join lateral jsonb_each(pending.payload) as field + where jsonb_typeof(field.value) in ('string', 'number', 'boolean', 'null') + ), filtered_candidates as materialized ( + select + event_value.event_id, + event_value.event_key, + event_value.payload, + subscription.id as subscription_id, + subscription.task_key, + subscription.queue, + subscription.payload_fields + from event_values event_value + join pgconductor._private_custom_event_predicates predicate + on predicate.event_key = event_value.event_key + and predicate.field_name = event_value.field_name + and jsonb_typeof(predicate.value) = jsonb_typeof(event_value.value) + and predicate.value = event_value.value + join pgconductor._private_custom_event_subscriptions subscription + on subscription.id = predicate.subscription_id + and subscription.event_key = event_value.event_key + and subscription.field_count > 0 + join pgconductor._private_tasks task + on task.key = subscription.task_key + and task.queue = subscription.queue + group by + event_value.event_id, + event_value.event_key, + event_value.payload, + subscription.id, + subscription.task_key, + subscription.queue, + subscription.payload_fields, + subscription.field_count + having count(distinct predicate.field_name) = subscription.field_count + ), unfiltered_candidates as materialized ( + select + pending.id as event_id, + pending.event_key, + pending.payload, + subscription.id as subscription_id, + subscription.task_key, + subscription.queue, + subscription.payload_fields + from pending + join pgconductor._private_custom_event_subscriptions subscription + on subscription.event_key = pending.event_key + and subscription.field_count = 0 + join pgconductor._private_tasks task + on task.key = subscription.task_key + and task.queue = subscription.queue + ), candidates as materialized ( + select * from filtered_candidates + union all + select * from unfiltered_candidates + ), matches as materialized ( + select distinct + candidate.event_id, + candidate.task_key, + candidate.queue, + candidate.payload_fields, + candidate.event_key, + candidate.payload, + candidate.subscription_id + from candidates candidate + ), inserted_destinations as ( + insert into pgconductor._private_executions ( + id, + task_key, + queue, + payload, + source_event_id, + event_subscription_id + ) + select + pgconductor._private_portable_uuidv7(), + candidate_match.task_key, + candidate_match.queue, + jsonb_build_object( + 'event', candidate_match.event_key, + 'payload', pgconductor._private_extract_event_payload( + candidate_match.payload_fields, candidate_match.payload + ) ), - ', ' - ) || ')' - end; -$_$; - -create or replace function pgconductor._private_sync_database_trigger() - returns trigger - language plpgsql - security definer - set search_path to '' -as $_$ -declare - v_table_name text := coalesce(new.table_name, old.table_name); - v_schema_name text := coalesce(new.schema_name, old.schema_name); - v_op pgconductor._private_event_operation; - v_invoke_blocks text; - v_has_subscriptions boolean; -begin - -- Only process database event subscriptions (schema_name is not null) - if v_schema_name is null then - return coalesce(new, old); - end if; - - -- Process each operation type (insert, update, delete) - foreach v_op in array array['insert', 'update', 'delete']::pgconductor._private_event_operation[] loop - -- Drop existing trigger and function - execute format( - 'drop trigger if exists pgconductor_event_%s on %I.%I', - v_op::text, v_schema_name, v_table_name - ); - - execute format( - 'drop function if exists pgconductor._private_trigger_event_%s_on_%I_%I', - v_op::text, v_schema_name, v_table_name - ); - - -- Check if there are any subscriptions for this operation - select exists( - select 1 - from pgconductor._private_event_subscriptions - where table_name = v_table_name - and schema_name = v_schema_name - and operation = v_op - ) into v_has_subscriptions; - - if v_has_subscriptions then - -- Build if blocks to check conditions and append to arrays - -- Each subscription's when_clause is evaluated inside the trigger function - v_invoke_blocks := ( - select string_agg(format( - $sql$ - if %s then - v_task_keys := array_append(v_task_keys, %L); - v_queues := array_append(v_queues, %L); - v_payloads := array_append(v_payloads, jsonb_build_object( - 'event', %L, - 'payload', jsonb_build_object( - 'old', case when tg_op is distinct from 'INSERT' then %s else null end, - 'new', case when tg_op is distinct from 'DELETE' then %s else null end, - 'tg_table', tg_table_name, - 'tg_op', tg_op - ) - )); - v_subscription_ids := array_append(v_subscription_ids, %L); - end if; - $sql$, - coalesce(nullif(sub.when_clause, ''), 'true'), - sub.task_key, - t.queue, - format('%s.%s.%s', v_schema_name, v_table_name, v_op::text), - pgconductor._private_build_column_list(sub.column_names, 'old'), - pgconductor._private_build_column_list(sub.column_names, 'new'), - sub.id - ), e'\n') - from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue - where sub.table_name = v_table_name - and sub.schema_name = v_schema_name - and sub.operation = v_op - ); - - -- Create trigger function - execute format( - $sql$ - create or replace function pgconductor._private_trigger_event_%s_on_%I_%I() - returns trigger - language plpgsql - security definer - set search_path to '' - as $inner$ - declare - v_task_keys text[]; - v_queues text[]; - v_payloads jsonb[]; - v_subscription_ids uuid[]; - begin - %s - - if array_length(v_task_keys, 1) > 0 then - insert into pgconductor._private_executions (task_key, queue, payload, subscription_id) - select unnest(v_task_keys), unnest(v_queues), unnest(v_payloads), unnest(v_subscription_ids); - end if; - - if tg_op = 'DELETE' then - return old; - end if; - - return new; - end - $inner$ - $sql$, - v_op::text, - v_schema_name, - v_table_name, - v_invoke_blocks - ); - - -- Create trigger - execute format( - $sql$ - create trigger pgconductor_event_%s - after %s on %I.%I - for each row - execute function pgconductor._private_trigger_event_%s_on_%I_%I() - $sql$, - v_op::text, - upper(v_op::text), - v_schema_name, - v_table_name, - v_op::text, - v_schema_name, - v_table_name - ); - end if; - end loop; - - if tg_op = 'DELETE' then - return old; - end if; - - return new; -end; -$_$; - -create trigger sync_database_trigger - after insert or delete or update on pgconductor._private_event_subscriptions - for each row - execute function pgconductor._private_sync_database_trigger(); + candidate_match.event_id, + candidate_match.subscription_id + from matches candidate_match + on conflict (source_event_id, event_subscription_id, queue) + where source_event_id is not null and event_subscription_id is not null + do nothing + returning source_event_id + ), write_barrier as ( + select true as ready from inserted_destinations + union all + select true + where not exists (select 1 from inserted_destinations) + ), updated as ( + update pgconductor._private_custom_events source_event + set dispatched_at = pgconductor._private_current_time() + from pending + cross join write_barrier + where source_event.id = pending.id + and source_event.dispatched_at is null + returning source_event.id + ) + select claimed.id as event_id + from claimed + left join updated + on updated.id = claimed.id + where claimed.dispatched_at is not null + or updated.id is not null; +$function$; create or replace function pgconductor.emit_event( p_event_key text, p_payload jsonb default '{}'::jsonb ) - returns uuid - language sql - volatile - set search_path to '' -as $_$ - insert into pgconductor._private_custom_events (event_key, payload) - values (p_event_key, p_payload) - returning id; -$_$; +returns uuid +language sql +volatile +set search_path to '' +as $function$ + with inserted_event as ( + insert into pgconductor._private_custom_events (id, event_key, payload) + values (pgconductor._private_portable_uuidv7(), p_event_key, coalesce(p_payload, '{}'::jsonb)) + returning id + ), inserted_source as ( + insert into pgconductor._private_executions ( + id, task_key, queue, payload + ) + select + inserted_event.id, + 'pgconductor.event-dispatch', + 'pgconductor.internal', + '{}'::jsonb + from inserted_event + returning id + ) + select id from inserted_source; +$function$; + +create or replace function pgconductor._private_remove_custom_events( + p_before timestamptz, + p_batch integer +) +returns integer +language plpgsql +volatile +security definer +set search_path to '' +as $function$ +declare + v_removed integer; +begin + with candidates as materialized ( + select event.id + from pgconductor._private_custom_events event + left join pgconductor._private_executions source + on source.id = event.id + and source.queue = 'pgconductor.internal' + and source.task_key = 'pgconductor.event-dispatch' + where event.created_at < p_before + and ( + source.id is null + or source.completed_at is not null + or source.failed_at is not null + or source.cancelled + ) + order by event.created_at, event.event_position + limit greatest(coalesce(p_batch, 0), 0) + for update of event skip locked + ) + delete from pgconductor._private_custom_events event + using candidates + where event.id = candidates.id; + + get diagnostics v_removed = row_count; + return v_removed; +end; +$function$; `, }); diff --git a/packages/pgconductor-js/src/maintenance-task.ts b/packages/pgconductor-js/src/maintenance-task.ts index 0ab6a78..325b5de 100644 --- a/packages/pgconductor-js/src/maintenance-task.ts +++ b/packages/pgconductor-js/src/maintenance-task.ts @@ -37,7 +37,19 @@ export const createMaintenanceTask = (queue: Q }, async (_, ctx) => { const { db, tasks, signal } = ctx; - // Skip if no tasks have retention settings (check in-memory config) + + // Custom event logs have a fixed seven-day retention policy. The + // always-present internal worker performs this global cleanup. + if (queue === "pgconductor.internal") { + const now = await db.getCurrentTime({ signal }); + const before = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); + let hasMoreEvents = true; + while (hasMoreEvents) { + hasMoreEvents = await db.removeCustomEvents(before, BATCH_SIZE, { signal }); + } + } + + // Skip execution cleanup when no tasks have retention settings (check in-memory config) const hasRetention = Array.from(tasks.values()).some( (t) => t.removeOnComplete || t.removeOnFail, ); diff --git a/packages/pgconductor-js/src/orchestrator.ts b/packages/pgconductor-js/src/orchestrator.ts index 8f68a64..87e4115 100644 --- a/packages/pgconductor-js/src/orchestrator.ts +++ b/packages/pgconductor-js/src/orchestrator.ts @@ -1,4 +1,4 @@ -import { Worker, type WorkerConfig } from "./worker"; +import { createEventDispatchTask, EVENT_DISPATCH_QUEUE, Worker, type WorkerConfig } from "./worker"; import { DatabaseClient } from "./database-client"; import { MigrationStore } from "./migration-store"; import { SchemaManager } from "./schema-manager"; @@ -13,14 +13,14 @@ import { noop } from "./lib/noop"; import { coerceError } from "./lib/coerce-error"; export type OrchestratorOptions = { - conductor: Conductor; + conductor: Conductor; tasks?: ValidateTasksQueue<"default", TTasks>; defaultWorker?: Partial; workers?: Worker[]; }; type InternalOrchestratorOptions = { - conductor: Conductor; + conductor: Conductor; tasks?: readonly AnyTask[]; defaultWorker?: Partial; workers?: Worker[]; @@ -39,6 +39,7 @@ const STALE_ORCHESTRATOR_MAX_AGE_MS = HEARTBEAT_INTERVAL_MS * 10; export class Orchestrator { private readonly db: DatabaseClient; private readonly workers: Worker[] = []; + private readonly eventWorker: Worker; private readonly orchestratorId: string; private readonly migrationStore: MigrationStore; private readonly schemaManager: SchemaManager; @@ -69,17 +70,47 @@ export class Orchestrator { this.logger, options.defaultWorker, options.conductor.options.context, + options.conductor.options.events?.definitions ?? [], + true, + options.conductor.options.events?.hasTypeOnlyDefinitions ?? false, ); this.workers.push(worker); } for (const w of options.workers || []) { - if (this.workers.find((existing) => existing.queueName === w.queueName)) { - throw new Error(`Duplicate worker name: ${w.queueName}`); - } - this.workers.push(w); } + + const queues = new Set(); + for (const worker of this.workers) { + if (worker.queueName === EVENT_DISPATCH_QUEUE) { + throw new Error(`Queue "${EVENT_DISPATCH_QUEUE}" is reserved for internal use`); + } + if (queues.has(worker.queueName)) { + throw new Error( + `Orchestrator cannot configure multiple workers for queue "${worker.queueName}"; configure one worker with all tasks for that queue`, + ); + } + queues.add(worker.queueName); + } + + this.eventWorker = new Worker( + EVENT_DISPATCH_QUEUE, + [createEventDispatchTask()], + this.db, + this.logger, + { + concurrency: 1, + fetchBatchSize: 10, + flushBatchSize: 10, + pollIntervalMs: options.defaultWorker?.pollIntervalMs || 1000, + flushIntervalMs: options.defaultWorker?.flushIntervalMs || 2000, + }, + {}, + [], + true, + false, + ); } static create[]>( @@ -139,6 +170,9 @@ export class Orchestrator { } this._stopDeferred = new Deferred(); + // Startup can fail before callers have a reason to observe `stopped`. + // Keep the rejection available to explicit awaiters while marking it handled. + this._stopDeferred.promise.catch(noop); this._startDeferred = new Deferred(); this._abortController = new AbortController(); this.registerSignalHandlers(); @@ -189,32 +223,59 @@ export class Orchestrator { // Start heartbeat loop this.startHeartbeatLoop(); - // Kick off all workers (don't await yet!) - if (runOnce) { - // Drain mode: workers will process and stop - this.workers.forEach((w) => void w.drain(this.orchestratorId)); - } else { - // Normal mode: workers will run continuously - this.workers.forEach((w) => void w.run(this.orchestratorId)); - } - - // Wait for ALL workers to finish starting (register() complete) - await Promise.all(this.workers.map((w) => w.started)); + const startWorkers = async () => { + const userWorkerLifecycles = this.workers.map((worker) => { + const lifecycle = runOnce + ? worker.drain(this.orchestratorId) + : worker.run(this.orchestratorId); + // A sibling may fail registration before these promises are returned + // to the caller. Mark every lifecycle as observed immediately. + lifecycle.catch(noop); + return lifecycle; + }); + + // Subscription registration must commit before the dispatcher can + // claim source executions and freeze their delivery snapshot. + await Promise.all(this.workers.map((worker) => worker.started)); + + const eventWorkerLifecycle = runOnce + ? this.eventWorker.drain(this.orchestratorId) + : this.eventWorker.run(this.orchestratorId); + eventWorkerLifecycle.catch(noop); + await this.eventWorker.started; + + return { + allWorkers: Promise.all([...userWorkerLifecycles, eventWorkerLifecycle]), + }; + }; + + let { allWorkers } = await startWorkers(); // NOW signal that orchestrator has started this.startDeferred.resolve(); - // Wait for shutdown signal or all workers to complete - await Promise.race([ - Promise.all(this.workers.map((w) => w.stopped)), - this.waitForShutdownSignal(), - ]); + if (runOnce) { + // Event fan-out can create work in a queue that already finished its + // pass, and destination tasks can recursively emit more events. + await allWorkers; + while ( + this.eventWorker.drainDidWork || + this.workers.some((worker) => worker.drainDidWork) + ) { + ({ allWorkers } = await startWorkers()); + await allWorkers; + } + } else { + // Wait for shutdown signal or all workers to complete + await Promise.race([allWorkers, this.waitForShutdownSignal()]); + } // Stop gracefully await this.stopWorkers(); } catch (err) { error = coerceError(err); this.logger.error(err); + await this.stopWorkers(); // Only reject startDeferred if startup hasn't completed yet if (!this.startDeferred.isSettled) { @@ -329,9 +390,10 @@ export class Orchestrator { signal.signal_payload && signal.signal_payload.queue ) { - const worker = this.workers.find( - (w) => w.queueName === signal.signal_payload?.queue, - ); + const worker = + signal.signal_payload.queue === EVENT_DISPATCH_QUEUE + ? this.eventWorker + : this.workers.find((w) => w.queueName === signal.signal_payload?.queue); if (worker) { worker.cancelExecutions([signal.signal_execution_id]); } @@ -373,7 +435,7 @@ export class Orchestrator { * Stop all workers gracefully */ private async stopWorkers(): Promise { - await Promise.all(this.workers.map((worker) => worker.stop())); + await Promise.all([...this.workers.map((worker) => worker.stop()), this.eventWorker.stop()]); } /** diff --git a/packages/pgconductor-js/src/query-builder.ts b/packages/pgconductor-js/src/query-builder.ts index efee09b..62fefab 100644 --- a/packages/pgconductor-js/src/query-builder.ts +++ b/packages/pgconductor-js/src/query-builder.ts @@ -44,6 +44,16 @@ export type RemoveExecutionsArgs = { batchSize: number; }; +export type RemoveCustomEventsArgs = { + before: Date; + batchSize: number; +}; + +export type DispatchCustomEventsArgs = { + eventIds: string[]; + orchestratorId: string; +}; + export type RegisterWorkerArgs = { queueName: string; taskSpecs: TaskSpec[]; @@ -374,12 +384,14 @@ export class QueryBuilder { returning e.id, e.task_key, e.queue, e.payload, e.waiting_on_execution_id, e.waiting_step_key, e.cancelled, e.last_error, e.dedupe_key, e.cron_expression, e.locked_by, e."group", e.priority, e.run_at, e.created_at, + e.source_event_id, e.event_subscription_id, e.dead_letter_source_execution_id, e.dead_letter_source_queue, e.dead_letter_source_task_key, e.dead_letter_error, e.dead_letter_attempts, e.dead_letter_failed_at ) select id, task_key, queue, payload, waiting_on_execution_id, waiting_step_key, cancelled, last_error, dedupe_key, cron_expression, locked_by, "group", + source_event_id, event_subscription_id, dead_letter_source_execution_id, dead_letter_source_queue, dead_letter_source_task_key, dead_letter_error, dead_letter_attempts, dead_letter_failed_at from claimed @@ -714,6 +726,18 @@ export class QueryBuilder { `; } + buildRemoveCustomEvents({ + before, + batchSize, + }: RemoveCustomEventsArgs): PendingQuery<{ deleted_count: number }[]> { + return this.sql<{ deleted_count: number }[]>` + select pgconductor._private_remove_custom_events( + ${before.toISOString()}::timestamptz, + ${batchSize}::integer + )::integer as deleted_count + `; + } + buildRegisterWorker({ queueName, taskSpecs, @@ -751,14 +775,9 @@ export class QueryBuilder { const eventSubscriptionRows = eventSubscriptions.map((spec) => ({ task_key: spec.task_key, - queue: spec.queue, event_key: spec.event_key, - schema_name: spec.schema_name, - table_name: spec.table_name, - operation: spec.operation, - when_clause: spec.when_clause, payload_fields: spec.payload_fields, - column_names: spec.column_names, + filter: spec.filter, })); return this.sql` @@ -1067,6 +1086,19 @@ export class QueryBuilder { `; } + buildDispatchCustomEvents({ + eventIds, + orchestratorId, + }: DispatchCustomEventsArgs): PendingQuery<{ event_id: string }[]> { + return this.sql<{ event_id: string }[]>` + select event_id + from pgconductor._private_dispatch_custom_events( + ${this.sql.array(eventIds)}::uuid[], + ${orchestratorId}::uuid + ) + `; + } + buildEmitEvent({ eventKey, payload }: EmitEventArgs): PendingQuery<{ id: string }[]> { return this.sql<{ id: string }[]>` select pgconductor.emit_event( diff --git a/packages/pgconductor-js/src/schemas.ts b/packages/pgconductor-js/src/schemas.ts index 749a4a7..7e0aa83 100644 --- a/packages/pgconductor-js/src/schemas.ts +++ b/packages/pgconductor-js/src/schemas.ts @@ -1,5 +1,5 @@ import type { TaskDefinition } from "./task-definition"; -import type { EventDefinition, GenericDatabase } from "./event-definition"; +import type { EventDefinition } from "./event-definition"; /** * Stackable adapter for task definitions. @@ -65,89 +65,50 @@ export class TaskSchemas< * .fromUnion() */ export class EventSchemas< - TSchemaTypes extends readonly EventDefinition[] = readonly [], - TUnionTypes extends EventDefinition = never, + TSchemaTypes extends readonly EventDefinition[] = readonly [], + TUnionTypes extends EventDefinition = never, > { - private constructor(readonly definitions: TSchemaTypes) {} + private constructor( + readonly definitions: TSchemaTypes, + readonly hasTypeOnlyDefinitions: boolean, + ) {} /** * Create EventSchemas from standard-schema based event definitions. */ - static fromSchema[]>( + static fromSchema[]>( events: T, ): EventSchemas { - return new EventSchemas(events); + return new EventSchemas(events, false); } /** * Add type-only event definitions via union type. */ - static fromUnion>(): EventSchemas< + static fromUnion>(): EventSchemas< readonly [], TUnion > { - return new EventSchemas([]); + return new EventSchemas([], true); } /** * Chain: Add more standard-schema based event definitions. */ - fromSchema[]>( + fromSchema[]>( events: T, ): EventSchemas { - return new EventSchemas([...this.definitions, ...events]); + return new EventSchemas([...this.definitions, ...events], this.hasTypeOnlyDefinitions); } /** * Chain: Add type-only event definitions via union type. */ - fromUnion>(): EventSchemas< + fromUnion>(): EventSchemas< TSchemaTypes, TUnionTypes | TUnion > { - return new EventSchemas(this.definitions); - } -} - -type SupabaseDatabase = { - [schema_name: string]: { - Tables: { - [table_name: string]: { - Row: unknown; - Insert?: unknown; - Update?: unknown; - }; - }; - }; -}; - -type ConvertSupabaseTables = { - [Schema in keyof T]: T[Schema] extends { Tables: infer Tables } - ? { - [Table in keyof Tables]: Tables[Table] extends { Row: infer Row } ? Row : unknown; - } - : {}; -}; - -/** - * Adapter for database type definitions. - * Wraps database types to allow different sources in the future (e.g., Supabase). - */ -export class DatabaseSchema<_TDatabase extends GenericDatabase> { - private constructor() {} - - /** - * Create DatabaseSchema from generated types (e.g., from pgtyped, kysely, etc.). - */ - static fromGeneratedTypes(): DatabaseSchema { - return new DatabaseSchema(); - } - - /** - * Create DatabaseSchema from Supabase generated types. - */ - static fromSupabaseTypes(): DatabaseSchema> { - return new DatabaseSchema(); + return new EventSchemas(this.definitions, true); } } @@ -165,5 +126,3 @@ export type InferEventsFromSchema = ? TSchema : readonly [...TSchema, TUnion] : readonly []; - -export type InferDatabaseFromSchema = T extends DatabaseSchema ? U : GenericDatabase; diff --git a/packages/pgconductor-js/src/select-columns.ts b/packages/pgconductor-js/src/select-columns.ts index 767585c..e2b96ac 100644 --- a/packages/pgconductor-js/src/select-columns.ts +++ b/packages/pgconductor-js/src/select-columns.ts @@ -115,29 +115,23 @@ type FirstError = T extends [infer Head, ...infer Tail : undefined : undefined; +type FirstDuplicate = T extends [infer Head, ...infer Tail] + ? Head extends Tail[number] + ? Head + : FirstDuplicate + : never; + type PickFromKeys = { [K in Keys]: Obj[K] }; type ValidatePart = Trim extends infer P extends string ? P extends "" ? `[Column Selection Error]: Empty entry. Remove trailing commas or extra spaces.` - : // quoted identifier - P extends `"${infer Body}"` - ? // forbid raw " inside quoted body - Body extends `${string}"${string}` - ? `[Column Selection Error]: Invalid quote in "${P}". Use "" to escape quotes inside identifiers.` - : Body extends keyof Obj - ? Body - : `[Column Selection Error]: Column "${Body}" does not exist.` - : // malformed quoting - P extends `${string}"${string}` - ? `[Column Selection Error]: Malformed quotes in "${P}". Quoted identifiers must be fully wrapped.` - : // unquoted identifier - IsValidUnquoted

extends false - ? `[Column Selection Error]: Invalid identifier "${P}". Must start with a letter or underscore.` - : P extends keyof Obj - ? P - : `[Column Selection Error]: Column "${P}" does not exist.` + : IsValidUnquoted

extends false + ? `[Column Selection Error]: Invalid identifier "${P}". Must start with a letter or underscore.` + : P extends keyof Obj + ? P + : `[Column Selection Error]: Column "${P}" does not exist.` : never; type ValidateParts = { @@ -151,7 +145,11 @@ export type ParseSelection = ? FirstError extends string ? ColumnSelectionError> : never - : PickFromKeys, Obj> + : FirstDuplicate extends infer Duplicate extends string + ? [Duplicate] extends [never] + ? PickFromKeys, Obj> + : ColumnSelectionError<`[Column Selection Error]: Duplicate column "${Duplicate}".`> + : never : never : never; diff --git a/packages/pgconductor-js/src/task-context.ts b/packages/pgconductor-js/src/task-context.ts index 14fc735..8f04b3b 100644 --- a/packages/pgconductor-js/src/task-context.ts +++ b/packages/pgconductor-js/src/task-context.ts @@ -104,7 +104,11 @@ export class TaskContext< any, string >[], - Events extends readonly EventDefinition[] = readonly EventDefinition[], + Events extends readonly EventDefinition[] = readonly EventDefinition< + string, + any, + any + >[], > { private readonly windowChecker?: WindowChecker; @@ -116,7 +120,7 @@ export class TaskContext< static create< Tasks extends readonly TaskDefinition[], - Events extends readonly EventDefinition[], + Events extends readonly EventDefinition[], Extra extends object, >(opts: TaskContextOptions, extra?: Extra): TaskContext & Extra { const base = new TaskContext(opts); diff --git a/packages/pgconductor-js/src/task-definition.ts b/packages/pgconductor-js/src/task-definition.ts index cf7257b..90cb2bf 100644 --- a/packages/pgconductor-js/src/task-definition.ts +++ b/packages/pgconductor-js/src/task-definition.ts @@ -1,4 +1,11 @@ import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { + EventDefinition, + FilterForEvent, + FindEventByIdentifier, + InferEventPayload, +} from "./event-definition"; +import type { ValidateColumns } from "./select-columns"; type ObjectSchema = StandardSchemaV1; @@ -123,28 +130,53 @@ export type CronTrigger = { cron: string; name: string; group?: string }; // Event trigger - triggers when a custom event is emitted export type CustomEventTrigger< TName extends string = string, - TFields extends string | undefined = undefined, + TFields extends string | undefined = string | undefined, + TFilter extends Record | undefined = Record | undefined, > = { event: TName; - when?: string; fields?: TFields; + filter?: TFilter; }; -// Database event trigger - triggers on CDC events -export type DatabaseEventTrigger< - TSchema extends string = string, - TTable extends string = string, - TOp extends "insert" | "update" | "delete" = "insert" | "update" | "delete", - TColumns extends string = string, -> = { - schema: TSchema; - table: TTable; - operation: TOp; - when?: string; - columns: TColumns; -}; - -export type Trigger = InvocableTrigger | CronTrigger | CustomEventTrigger | DatabaseEventTrigger; +export type Trigger = InvocableTrigger | CronTrigger | CustomEventTrigger; + +type ValidateEventFields = T extends { fields: infer Fields extends string } + ? ValidateColumns> extends Fields + ? T + : ValidateColumns> + : T; + +type ValidateEventFilter = T extends { filter: infer Filter } + ? Filter extends FilterForEvent + ? Exclude> extends never + ? T + : `Filter for event "${Name}" contains undeclared fields.` + : `Filter for event "${Name}" contains undeclared or incorrectly typed fields.` + : T; + +type ValidateCustomEventTrigger< + Events extends readonly EventDefinition[], + T, +> = T extends { event: infer Name extends string } + ? Events extends readonly [] + ? T + : FindEventByIdentifier extends infer Event + ? [Event] extends [never] + ? `Event "${Name}" is not defined in the conductor event catalog.` + : ValidateEventFields extends infer FieldsResult + ? FieldsResult extends T + ? ValidateEventFilter + : FieldsResult + : never + : never + : T; + +export type ValidateEventTriggers< + Events extends readonly EventDefinition[], + TTriggers, +> = TTriggers extends readonly any[] + ? { [K in keyof TTriggers]: ValidateCustomEventTrigger } + : ValidateCustomEventTrigger; // Check if triggers include invocable export type HasInvocable = TTriggers extends readonly any[] @@ -164,36 +196,12 @@ export type HasCron = TTriggers extends readonly any[] ? true : false; -// Check if triggers include custom event +// Check if triggers include a custom event. export type HasCustomEvent = TTriggers extends readonly any[] - ? Extract extends infer Extracted - ? Extracted extends never - ? false - : Exclude extends never - ? false // All events are database events - : true // At least one custom event - : false - : TTriggers extends { event: string } - ? TTriggers extends { schema: string } - ? false // Database event, not custom event - : true - : false; - -// Check if triggers include database event -export type HasDatabaseEvent = TTriggers extends readonly any[] - ? { - schema: string; - table: string; - operation: "insert" | "update" | "delete"; - } extends TTriggers[number] + ? Extract extends never ? false - : Extract< - TTriggers[number], - { schema: string; table: string; operation: "insert" | "update" | "delete" } - > extends never - ? false - : true - : TTriggers extends { schema: string; table: string; operation: "insert" | "update" | "delete" } + : true + : TTriggers extends { event: string } ? true : false; @@ -229,8 +237,10 @@ export type ValidateTriggers< : TTriggers : `Triggers array cannot be empty. Provide at least one trigger.` : // Single trigger case - TTriggers extends InvocableTrigger - ? TaskIdentifierIsDefined extends true - ? TTriggers - : `Task "${TName}" of queue "${TQueue}" is not defined in the conductor catalog. Remove { invocable: true } from triggers, or add a task definition to the conductor's tasks array.` - : TTriggers; + TTriggers extends Trigger + ? TTriggers extends InvocableTrigger + ? TaskIdentifierIsDefined extends true + ? TTriggers + : `Task "${TName}" of queue "${TQueue}" is not defined in the conductor catalog. Remove { invocable: true } from triggers, or add a task definition to the conductor's tasks array.` + : TTriggers + : "Invalid trigger. Use an invocable, cron, or custom event trigger."; diff --git a/packages/pgconductor-js/src/task.ts b/packages/pgconductor-js/src/task.ts index b611feb..85c6e6b 100644 --- a/packages/pgconductor-js/src/task.ts +++ b/packages/pgconductor-js/src/task.ts @@ -4,19 +4,9 @@ import type { HasInvocable, HasCron, HasCustomEvent, - HasDatabaseEvent, CronTrigger, } from "./task-definition"; -import type { - EventDefinition, - FindEventByIdentifier, - InferEventPayload, - GenericDatabase, - DatabaseEventPayload, - SchemaName, - TableName, - RowType, -} from "./event-definition"; +import type { EventDefinition, FindEventByIdentifier, InferEventPayload } from "./event-definition"; import type { SelectedRow } from "./select-columns"; import * as assert from "./lib/assert"; @@ -100,29 +90,10 @@ type ExtractCronTriggers = TTriggers extends readonly any[] ? TTriggers : never; -// Extract custom event triggers from array +// Extract custom event triggers from array. type ExtractCustomEventTriggers = TTriggers extends readonly any[] - ? TTriggers[number] extends infer T - ? T extends { event: string } - ? T extends { schema: string } - ? never // Database event, not custom event - : T - : never - : never + ? Extract : TTriggers extends { event: string } - ? TTriggers extends { schema: string } - ? never // Database event, not custom event - : TTriggers - : never; - -// Extract database event triggers from array -type ExtractDatabaseEventTriggers = TTriggers extends readonly any[] - ? TTriggers[number] extends infer T - ? T extends { schema: string; table: string; operation: "insert" | "update" | "delete" } - ? T - : never - : never - : TTriggers extends { schema: string; table: string; operation: "insert" | "update" | "delete" } ? TTriggers : never; @@ -135,11 +106,11 @@ type CronEventUnion = : never; // Build custom event union from triggers -type CustomEventUnion[]> = +type CustomEventUnion[]> = ExtractCustomEventTriggers extends infer T ? T extends { event: infer TName extends string } ? FindEventByIdentifier extends infer TEvent - ? TEvent extends EventDefinition + ? TEvent extends EventDefinition ? T extends { fields: infer TFields extends string } ? { name: TName; @@ -151,64 +122,17 @@ type CustomEventUnion = - ExtractDatabaseEventTriggers extends infer T - ? T extends { schema: infer TSchema extends string } - ? T extends { table: infer TTable extends string } - ? T extends { operation: infer TOp extends "insert" | "update" | "delete" } - ? TSchema extends SchemaName - ? TTable extends TableName - ? T extends { columns: infer TColumns extends string } - ? { - name: `${TSchema}.${TTable}.${TOp}`; - payload: DatabaseEventPayload< - RowType, - TOp, - TColumns - >; - } - : never - : T extends { columns: infer _TColumns extends string } - ? { - name: `${TSchema}.${TTable}.${TOp}`; - payload: { - old: TOp extends "delete" | "update" ? Record : null; - new: TOp extends "insert" | "update" ? Record : null; - tg_table: string; - tg_op: Uppercase; - }; - } - : never - : T extends { columns: infer _TColumns extends string } - ? { - name: `${TSchema}.${TTable}.${TOp}`; - payload: { - old: TOp extends "delete" | "update" ? Record : null; - new: TOp extends "insert" | "update" ? Record : null; - tg_table: string; - tg_op: Uppercase; - }; - } - : never - : never - : never - : never - : never; - // Conditional event type based on triggers export type TaskEventFromTriggers< TTriggers, TPayload extends object, - Events extends readonly EventDefinition[] = [], - Database extends GenericDatabase = {}, + Events extends readonly EventDefinition[] = [], > = | (HasInvocable extends true ? { name: "pgconductor.invoke"; payload: TPayload } : never) | (HasCron extends true ? CronEventUnion : never) - | (HasCustomEvent extends true ? CustomEventUnion : never) - | (HasDatabaseEvent extends true ? DatabaseEventUnion : never); + | (HasCustomEvent extends true ? CustomEventUnion : never); // Conditional execute function type based on whether task has batch config export type ExecuteFunction< diff --git a/packages/pgconductor-js/src/versions.ts b/packages/pgconductor-js/src/versions.ts index 3273eeb..0699e59 100644 --- a/packages/pgconductor-js/src/versions.ts +++ b/packages/pgconductor-js/src/versions.ts @@ -1,3 +1,3 @@ /* This file is auto-generated by `just build-migrations`; DO NOT EDIT */ export const PACKAGE_VERSION = "0.1.0"; -export const MIGRATION_NUMBER = 2; +export const MIGRATION_NUMBER = 1; diff --git a/packages/pgconductor-js/src/worker.ts b/packages/pgconductor-js/src/worker.ts index 2274566..cd1a84b 100644 --- a/packages/pgconductor-js/src/worker.ts +++ b/packages/pgconductor-js/src/worker.ts @@ -10,7 +10,7 @@ import type { ExecutionReleased, ExecutionInvokeChild, } from "./database-client"; -import type { AnyTask, BatchConfig } from "./task"; +import { Task, type AnyTask, type BatchConfig } from "./task"; import type { TaskDefinition } from "./task-definition"; import { waitFor } from "./lib/wait-for"; import { mapConcurrent } from "./lib/map-concurrent"; @@ -30,6 +30,7 @@ import * as assert from "./lib/assert"; import { createMaintenanceTask } from "./maintenance-task"; import { makeChildLogger, type Logger } from "./lib/logger"; import type { EventDefinition } from "./event-definition"; +import { compileEventTriggers } from "./event-trigger-validation"; import { coerceError } from "./lib/coerce-error"; import type { TypedAbortController } from "./lib/typed-abort-controller"; @@ -47,6 +48,26 @@ export type WorkerConfig = { /** * The default configuration for the Worker. */ +export const EVENT_DISPATCH_QUEUE = "pgconductor.internal"; +export const EVENT_DISPATCH_TASK = "pgconductor.event-dispatch"; +const EVENT_DISPATCH_BATCH_SIZE = 10; + +export function createEventDispatchTask(): AnyTask { + return new Task( + { + name: EVENT_DISPATCH_TASK, + queue: EVENT_DISPATCH_QUEUE, + maxAttempts: 3, + removeOnComplete: true, + batch: { size: EVENT_DISPATCH_BATCH_SIZE, timeoutMs: 10 }, + }, + { invocable: true }, + async () => { + throw new Error("Event dispatch must be executed by the internal worker"); + }, + ); +} + export const DEFAULT_WORKER_CONFIG: WorkerConfig = { concurrency: 1, flushBatchSize: 2, @@ -124,7 +145,11 @@ export class Worker< any, string >[], - Events extends readonly EventDefinition[] = readonly EventDefinition[], + Events extends readonly EventDefinition[] = readonly EventDefinition< + string, + any, + any + >[], > { private orchestratorId: string | null = null; @@ -140,6 +165,7 @@ export class Worker< private _startDeferred: Deferred | null = null; private _stopDeferred: Deferred | null = null; private _abortController: AbortController | null = null; + private _drainDidWork = false; private _runningTasks = new Map>(); constructor( @@ -149,15 +175,19 @@ export class Worker< private readonly logger: Logger, config: Partial = {}, private readonly extraContext: object = {}, + private readonly eventDefinitions: readonly EventDefinition[] = [], + includeMaintenance = true, + private readonly allowUnknownEvents = false, ) { - const maintenanceTask = createMaintenanceTask(this.queueName); - this.tasks = tasks.reduce( - (m, task) => { - m.set(task.name, task); - return m; - }, - new Map([[maintenanceTask.name, maintenanceTask]]), - ); + const initialTasks = new Map(); + if (includeMaintenance) { + const maintenanceTask = createMaintenanceTask(this.queueName); + initialTasks.set(maintenanceTask.name, maintenanceTask); + } + this.tasks = tasks.reduce((registered, task) => { + registered.set(task.name, task); + return registered; + }, initialTasks); const fullConfig = { ...DEFAULT_WORKER_CONFIG, ...config }; @@ -188,6 +218,11 @@ export class Worker< return this._stopDeferred?.promise || Promise.resolve(); } + /** @internal Whether the last run-once pass observed any work. */ + get drainDidWork(): boolean { + return this._drainDidWork; + } + /** * Start the worker. * Returns when startup is complete (registration done). @@ -226,13 +261,32 @@ export class Worker< } this.orchestratorId = orchestratorId; + this._drainDidWork = false; this._startDeferred = new Deferred(); this._stopDeferred = new Deferred(); this._abortController = new AbortController(); - // Sample before calculating or registering cron schedules. - await this.clock.start(this.abortController.signal); - await this.register(); + // Startup failures reject both lifecycle promises; callers must never + // observe a worker that started partially. + try { + // Sample before calculating or registering cron schedules. + await this.clock.start(this.abortController.signal); + await this.register(); + } catch (error) { + // Registration is part of startup, not a running pipeline. Resolve the + // stop promise so callers that only await `start()` do not get an + // unhandled rejection, then discard every piece of this failed attempt. + const stopDeferred = this._stopDeferred; + this._abortController.abort(); + // Reject `started` so an Orchestrator observes registration failure, but + // attach a noop handler because callers that only use start() do not + // necessarily observe this internal lifecycle promise. + this._startDeferred.promise.catch(() => {}); + this._startDeferred.reject(error); + if (!stopDeferred.isSettled) stopDeferred.resolve(); + this.resetLifecycle(); + throw error; + } // Worker is now started this._startDeferred.resolve(); @@ -247,27 +301,49 @@ export class Worker< } const queue = new BatchingAsyncQueue(this.fetchBatchSize * 2, batchConfigs); - void this.fetchExecutions(queue, { runOnce }); - - (async () => { - try { - // Consume from queue → execute → flush - await this.flushResults(this.executeTasks(queue)); - } catch (err) { - this.logger.error("Worker pipeline error:", err); - } finally { - queue.close(); - this.clock.stop(); - this._stopDeferred?.resolve(); - this._startDeferred = null; - this._stopDeferred = null; - this._abortController = null; - } - })(); + if (runOnce) { + void this.runDrainPipeline(queue); + } else { + void this.fetchExecutions(queue, { runOnce }); + void (async () => { + try { + await this.flushResults(this.executeTasks(queue)); + } catch (error) { + this.logger.error("Worker pipeline error:", error); + this._stopDeferred?.reject(error); + } finally { + queue.close(); + if (this._stopDeferred && !this._stopDeferred.isSettled) this._stopDeferred.resolve(); + this.resetLifecycle(); + } + })(); + } return this._startDeferred.promise; } + private async runDrainPipeline(queue: BatchingAsyncQueue): Promise { + try { + const fetched = this.fetchExecutions(queue, { runOnce: true }); + await this.flushResults(this.executeTasks(queue)); + this._drainDidWork = (await fetched) > 0; + } catch (error) { + this.logger.error("Worker pipeline error:", error); + this._stopDeferred?.reject(error); + } finally { + if (this._stopDeferred && !this._stopDeferred.isSettled) this._stopDeferred.resolve(); + this.resetLifecycle(); + } + } + + private resetLifecycle(): void { + this.clock.stop(); + this._startDeferred = null; + this._stopDeferred = null; + this._abortController = null; + this.orchestratorId = null; + } + /** * Stop the worker gracefully. * Returns when shutdown is complete. @@ -344,43 +420,14 @@ export class Worker< }), ); - const eventSubscriptions: EventSubscriptionSpec[] = allTasks.flatMap((task) => { - const customEvents = task.triggers - .filter((t) => "event" in t && typeof t.event === "string") - .map((trigger): EventSubscriptionSpec => { - const customTrigger = trigger as any; - return { - task_key: task.name, - queue: this.queueName, - event_key: customTrigger.event, - schema_name: null, - table_name: null, - operation: null, - when_clause: customTrigger.when || null, - payload_fields: customTrigger.fields?.split(",").map((f: string) => f.trim()) || null, - column_names: null, - }; - }); - - const dbEvents = task.triggers - .filter((t) => "schema" in t && "table" in t && "operation" in t) - .map((trigger): EventSubscriptionSpec => { - const dbTrigger = trigger as any; - return { - task_key: task.name, - queue: this.queueName, - event_key: null, - schema_name: dbTrigger.schema, - table_name: dbTrigger.table, - operation: dbTrigger.operation, - when_clause: dbTrigger.when || null, - payload_fields: null, - column_names: dbTrigger.columns?.split(",").map((c: string) => c.trim()) || null, - }; - }); - - return [...customEvents, ...dbEvents]; - }); + const eventSubscriptions: EventSubscriptionSpec[] = allTasks.flatMap((task) => + compileEventTriggers(task.triggers, this.eventDefinitions, this.allowUnknownEvents).map( + (spec) => ({ + task_key: task.name, + ...spec, + }), + ), + ); await this.db.registerWorker( { @@ -397,7 +444,8 @@ export class Worker< private async fetchExecutions( queue: BatchingAsyncQueue, { runOnce = false }: { runOnce?: boolean }, - ) { + ): Promise { + let fetched = 0; assert.ok(this.orchestratorId, "orchestratorId must be set when starting the pipeline"); // Pre-compute task metadata once @@ -460,6 +508,7 @@ export class Worker< } for (const exec of executions) { + fetched++; await queue.push(exec); // waits if full if (this.signal.aborted) break; } @@ -469,6 +518,7 @@ export class Worker< } queue.close(); + return fetched; } // --- Stage 2: Execute tasks concurrently --- @@ -521,6 +571,10 @@ export class Worker< return []; } + if (this.queueName === EVENT_DISPATCH_QUEUE && taskKey === EVENT_DISPATCH_TASK) { + return this.executeEventDispatchBatch(activeExecs); + } + // If task has batch config, always use batch execution (even for single items) if (task.batch) { return this.executeBatchTask(task, taskKey, activeExecs); @@ -541,6 +595,52 @@ export class Worker< } } + private async executeEventDispatchBatch(executions: Execution[]): Promise { + assert.ok(this.orchestratorId, "orchestratorId must be set while dispatching events"); + + try { + const dispatched = await this.db.dispatchCustomEvents( + { + eventIds: executions.map((execution) => execution.id), + orchestratorId: this.orchestratorId, + }, + { signal: this.signal }, + ); + const dispatchedIds = new Set(dispatched); + + return executions.map((execution) => { + if (!dispatchedIds.has(execution.id)) { + return { + execution_id: execution.id, + orchestrator_id: execution.locked_by, + queue: execution.queue, + task_key: execution.task_key, + status: "failed" as const, + error: "Event dispatch claim is no longer valid", + }; + } + return { + execution_id: execution.id, + orchestrator_id: execution.locked_by, + queue: execution.queue, + task_key: execution.task_key, + status: "completed" as const, + result: undefined, + }; + }); + } catch (error) { + const message = coerceError(error).message; + return executions.map((execution) => ({ + execution_id: execution.id, + orchestrator_id: execution.locked_by, + queue: execution.queue, + task_key: execution.task_key, + status: "failed" as const, + error: message, + })); + } + } + /** * Execute a single task execution. * @@ -567,12 +667,12 @@ export class Worker< const scheduleName = exec.dedupe_key?.split("::")[1] || "unknown"; taskEvent = { name: scheduleName }; } else if ( + exec.source_event_id != null && exec.payload && typeof exec.payload === "object" && - "event" in exec.payload && - exec.payload.event !== "pgconductor.invoke" + "event" in exec.payload ) { - // Event-triggered execution (custom event or db event) + // Event-triggered executions carry dedicated database identity. taskEvent = { name: exec.payload.event, payload: exec.payload.payload, @@ -693,10 +793,10 @@ export class Worker< const scheduleName = exec.dedupe_key?.split("::")[1] || "unknown"; return { name: scheduleName }; } else if ( + exec.source_event_id != null && exec.payload && typeof exec.payload === "object" && - "event" in exec.payload && - exec.payload.event !== "pgconductor.invoke" + "event" in exec.payload ) { return { name: exec.payload.event, diff --git a/packages/pgconductor-js/tests/integration/event-pipeline.test.ts b/packages/pgconductor-js/tests/integration/event-pipeline.test.ts new file mode 100644 index 0000000..4cff45a --- /dev/null +++ b/packages/pgconductor-js/tests/integration/event-pipeline.test.ts @@ -0,0 +1,743 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import postgres from "postgres"; +import { z } from "zod"; +import { Conductor } from "../../src/conductor"; +import { + DatabaseClient, + type EventSubscriptionSpec, + type JsonValue, +} from "../../src/database-client"; +import { defineEvent } from "../../src/event-definition"; +import { DefaultLogger } from "../../src/lib/logger"; +import { Orchestrator } from "../../src/orchestrator"; +import { EventSchemas, TaskSchemas } from "../../src/schemas"; +import { defineTask } from "../../src/task-definition"; +import { TestDatabasePool, type TestDatabase } from "../fixtures/test-database"; +import { waitForCondition } from "../test-utils"; + +const INTERNAL_QUEUE = "pgconductor.internal"; +const DISPATCH_TASK = "pgconductor.event-dispatch"; + +type CustomSubscription = { + taskKey: string; + eventKey: string; + filter?: Record; + payloadFields?: string[]; + maxAttempts?: number; +}; + +describe("event pipeline", () => { + let pool: TestDatabasePool; + const databases: TestDatabase[] = []; + + beforeAll(async () => { + pool = await TestDatabasePool.create(); + }, 60_000); + + afterEach(async () => { + await Promise.all(databases.map((database) => database.destroy())); + databases.length = 0; + }); + + afterAll(async () => { + await pool?.destroy(); + }); + + async function database(): Promise { + const db = await pool.child(); + databases.push(db); + await Conductor.create({ sql: db.sql, context: {} }).ensureInstalled(); + return db; + } + + async function registerSubscriptions( + db: TestDatabase, + subscriptions: CustomSubscription[], + ): Promise { + const eventSubscriptions: EventSubscriptionSpec[] = subscriptions.map((subscription) => ({ + task_key: subscription.taskKey, + event_key: subscription.eventKey, + payload_fields: subscription.payloadFields || null, + filter: subscription.filter || null, + })); + await db.client.registerWorker({ + queueName: "default", + taskSpecs: subscriptions.map((subscription) => ({ + key: subscription.taskKey, + queue: "default", + maxAttempts: subscription.maxAttempts || 3, + })), + cronSchedules: [], + eventSubscriptions, + }); + } + + async function claimEvent( + db: TestDatabase, + eventId: string, + orchestratorId = crypto.randomUUID(), + ): Promise { + const claimed = await db.client.getExecutions({ + orchestratorId, + queueName: INTERNAL_QUEUE, + batchSize: 10, + filterTaskKeys: [], + }); + expect(claimed.some((execution) => execution.id === eventId)).toBe(true); + return orchestratorId; + } + + test("uses delivery identity rather than payload shape for direct invocations", async () => { + const db = await database(); + const payload = z.object({ + event: z.string(), + payload: z.object({ value: z.string() }), + }); + const singleDefinition = defineTask({ name: "pipeline.direct-single", payload }); + const batchDefinition = defineTask({ name: "pipeline.direct-batch", payload }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([singleDefinition, batchDefinition]), + context: {}, + }); + const received: unknown[] = []; + const single = conductor.createTask( + { name: "pipeline.direct-single" }, + { invocable: true }, + async (event) => { + received.push(event); + }, + ); + const batch = conductor.createTask( + { name: "pipeline.direct-batch", batch: { size: 10, timeoutMs: 10 } }, + { invocable: true }, + async (events) => { + received.push(...events); + }, + ); + await conductor.invoke( + { name: "pipeline.direct-single" }, + { event: "customer-action", payload: { value: "single" } }, + ); + await conductor.invoke( + { name: "pipeline.direct-batch" }, + { event: "customer-action", payload: { value: "batch" } }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [single, batch], + defaultWorker: { pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.drain(); + + expect(received).toEqual([ + { + name: "pgconductor.invoke", + payload: { event: "customer-action", payload: { value: "single" } }, + }, + { + name: "pgconductor.invoke", + payload: { event: "customer-action", payload: { value: "batch" } }, + }, + ]); + }); + + test("emits a durable dispatch execution whose id is the event id", async () => { + const db = await database(); + const eventId = await db.client.emitEvent({ + eventKey: "pipeline.persisted", + payload: { value: "ready" }, + }); + const [source] = await db.sql< + { id: string; queue: string; task_key: string; payload: Record }[] + >` + select id, queue, task_key, payload + from pgconductor._private_executions + where id = ${eventId}::uuid and queue = ${INTERNAL_QUEUE} + `; + expect(source).toEqual({ + id: eventId, + queue: INTERNAL_QUEUE, + task_key: DISPATCH_TASK, + payload: {}, + }); + const [event] = await db.sql< + { + id: string; + event_key: string; + payload: Record; + dispatched_at: Date | null; + }[] + >` + select id, event_key, payload, dispatched_at + from pgconductor._private_custom_events + where id = ${eventId}::uuid + `; + expect(event).toEqual({ + id: eventId, + event_key: "pipeline.persisted", + payload: { value: "ready" }, + dispatched_at: null, + }); + }); + + test("does not serialize same-key emitters for the lifetime of caller transactions", async () => { + const db = await database(); + const firstSql = postgres(db.url, { max: 1 }); + const secondSql = postgres(db.url, { max: 1 }); + let releaseFirst!: () => void; + let markFirstInserted!: () => void; + const release = new Promise((resolve) => { + releaseFirst = resolve; + }); + const firstInserted = new Promise((resolve) => { + markFirstInserted = resolve; + }); + const firstTransaction = firstSql.begin(async (transaction) => { + await transaction`select pgconductor.emit_event('pipeline.concurrent', '{}'::jsonb)`; + markFirstInserted(); + await release; + }); + + try { + await firstInserted; + let timeout: Timer | undefined; + const secondEmission = secondSql` + select pgconductor.emit_event('pipeline.concurrent', '{}'::jsonb) as id + `; + const [second] = await Promise.race([ + secondEmission, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error("same-key emission was transaction-serialized")), + 2_000, + ); + }), + ]); + if (timeout) clearTimeout(timeout); + expect(second?.id).toBeString(); + } finally { + releaseFirst(); + await firstTransaction; + await Promise.all([firstSql.end(), secondSql.end()]); + } + }); + + test("retains the event log independently after its dispatch execution settles", async () => { + const db = await database(); + const eventId = await db.client.emitEvent({ eventKey: "pipeline.retained", payload: {} }); + const orchestratorId = await claimEvent(db, eventId); + expect(await db.client.dispatchCustomEvents({ eventIds: [eventId], orchestratorId })).toEqual([ + eventId, + ]); + await db.client.returnExecutions({ + count: 1, + orchestratorId, + completed: [ + { + execution_id: eventId, + orchestrator_id: orchestratorId, + queue: INTERNAL_QUEUE, + task_key: DISPATCH_TASK, + status: "completed", + }, + ], + failed: [], + released: [], + invokeChild: [], + taskKeys: new Set([DISPATCH_TASK]), + }); + const [state] = await db.sql<{ source_exists: boolean; event_dispatched: boolean }[]>` + select + exists(select 1 from pgconductor._private_executions where id = ${eventId}::uuid and queue = ${INTERNAL_QUEUE}) as source_exists, + exists(select 1 from pgconductor._private_custom_events where id = ${eventId}::uuid and dispatched_at is not null) as event_dispatched + `; + expect(state).toEqual({ source_exists: false, event_dispatched: true }); + }); + + test("retention does not race active retries and removes terminal undispatched events", async () => { + const db = await database(); + const oldTime = new Date(Date.now() - 10 * 24 * 60 * 60 * 1000); + await db.client.setFakeTime({ date: oldTime }); + const dispatchedId = await db.client.emitEvent({ eventKey: "pipeline.cleanup", payload: {} }); + const owner = await claimEvent(db, dispatchedId); + await db.client.dispatchCustomEvents({ eventIds: [dispatchedId], orchestratorId: owner }); + const undispatchedId = await db.client.emitEvent({ + eventKey: "pipeline.cleanup", + payload: {}, + }); + await db.sql` + update pgconductor._private_executions + set failed_at = pgconductor._private_current_time(), locked_by = null, locked_at = null + where id = ${undispatchedId}::uuid and queue = ${INTERNAL_QUEUE} + `; + await db.client.clearFakeTime(); + const before = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + + await db.client.removeCustomEvents(before, 10); + let events = await db.sql<{ id: string }[]>` + select id from pgconductor._private_custom_events + where id in (${dispatchedId}::uuid, ${undispatchedId}::uuid) + order by id + `; + expect(events.map((event) => event.id)).toEqual([dispatchedId]); + + await db.sql` + update pgconductor._private_executions + set failed_at = pgconductor._private_current_time(), locked_by = null, locked_at = null + where id = ${dispatchedId}::uuid and queue = ${INTERNAL_QUEUE} + `; + await db.client.removeCustomEvents(before, 10); + events = await db.sql<{ id: string }[]>` + select id from pgconductor._private_custom_events + where id in (${dispatchedId}::uuid, ${undispatchedId}::uuid) + `; + expect(events).toHaveLength(0); + }); + + test("matches typed scalar filters, distinguishes missing from null, and selects payload fields", async () => { + const db = await database(); + const event = defineEvent({ + name: "pipeline.order", + payload: z.object({ + status: z.enum(["paid", "trial", "cancelled"]), + region: z.string(), + attempt: z.number(), + active: z.boolean(), + coupon: z.string().nullable().optional(), + }), + filterable: ["status", "region", "attempt", "active", "coupon"], + }); + const filteredDefinition = defineTask({ name: "pipeline.filtered", payload: z.object({}) }); + const nullDefinition = defineTask({ name: "pipeline.null", payload: z.object({}) }); + const allDefinition = defineTask({ name: "pipeline.all", payload: z.object({}) }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([filteredDefinition, nullDefinition, allDefinition]), + events: EventSchemas.fromSchema([event]), + context: {}, + }); + const received: { task: string; payload: Record }[] = []; + const filtered = conductor.createTask( + { name: "pipeline.filtered" }, + { + event: "pipeline.order", + filter: { status: ["paid", "trial"], region: ["us"], attempt: [1], active: [true] }, + }, + async (receivedEvent) => { + received.push({ task: "filtered", payload: receivedEvent.payload }); + }, + ); + const explicitNull = conductor.createTask( + { name: "pipeline.null" }, + { event: "pipeline.order", filter: { coupon: [null] } }, + async (receivedEvent) => { + received.push({ task: "null", payload: receivedEvent.payload }); + }, + ); + const all = conductor.createTask( + { name: "pipeline.all" }, + { event: "pipeline.order", fields: "status, region, coupon" }, + async (receivedEvent) => { + received.push({ task: "all", payload: receivedEvent.payload }); + }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [filtered, explicitNull, all], + defaultWorker: { pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + // A persisted event can predate worker registration. The hidden dispatcher + // must not claim it until all user subscriptions are registered. + await conductor.emit("pipeline.order", { + status: "paid", + region: "us", + attempt: 1, + active: true, + coupon: null, + }); + await orchestrator.start(); + try { + await conductor.emit("pipeline.order", { + status: "trial", + region: "us", + attempt: 1, + active: true, + }); + await conductor.emit("pipeline.order", { + status: "paid", + region: "eu", + attempt: 1, + active: true, + coupon: "SAVE", + }); + await waitForCondition(() => received.length === 6); + expect(received.filter((item) => item.task === "filtered")).toHaveLength(2); + expect(received.filter((item) => item.task === "null")).toHaveLength(1); + expect(received.filter((item) => item.task === "all")).toEqual([ + { task: "all", payload: { status: "paid", region: "us", coupon: null } }, + { task: "all", payload: { status: "trial", region: "us" } }, + { task: "all", payload: { status: "paid", region: "eu", coupon: "SAVE" } }, + ]); + } finally { + await orchestrator.stop(); + } + }); + + test("writes destinations and dispatch state atomically and reuses a frozen snapshot", async () => { + const db = await database(); + await registerSubscriptions(db, [ + { taskKey: "pipeline.first", eventKey: "pipeline.snapshot", filter: { kind: ["first"] } }, + ]); + const eventId = await db.client.emitEvent({ + eventKey: "pipeline.snapshot", + payload: { kind: "first" }, + }); + const orchestratorId = await claimEvent(db, eventId); + const first = await db.client.dispatchCustomEvents({ eventIds: [eventId], orchestratorId }); + expect(first).toEqual([eventId]); + + await registerSubscriptions(db, [ + { taskKey: "pipeline.second", eventKey: "pipeline.snapshot", filter: { kind: ["first"] } }, + ]); + const second = await db.client.dispatchCustomEvents({ eventIds: [eventId], orchestratorId }); + expect(second).toEqual([eventId]); + const destinations = await db.sql<{ task_key: string }[]>` + select task_key + from pgconductor._private_executions + where source_event_id = ${eventId}::uuid + `; + expect([...destinations]).toEqual([{ task_key: "pipeline.first" }]); + + const noMatchId = await db.client.emitEvent({ + eventKey: "pipeline.none", + payload: {}, + }); + const noMatchOwner = await claimEvent(db, noMatchId, crypto.randomUUID()); + expect( + await db.client.dispatchCustomEvents({ eventIds: [noMatchId], orchestratorId: noMatchOwner }), + ).toEqual([noMatchId]); + const [noMatch] = await db.sql<{ dispatched_at: Date | null }[]>` + select dispatched_at + from pgconductor._private_custom_events + where id = ${noMatchId}::uuid + `; + expect(noMatch?.dispatched_at).not.toBeNull(); + }); + + test("rejects stale claims and rolls destination insertion back with dispatch state", async () => { + const db = await database(); + await registerSubscriptions(db, [{ taskKey: "pipeline.atomic", eventKey: "pipeline.atomic" }]); + const eventId = await db.client.emitEvent({ eventKey: "pipeline.atomic", payload: {} }); + const owner = await claimEvent(db, eventId); + expect( + await db.client.dispatchCustomEvents({ + eventIds: [eventId], + orchestratorId: crypto.randomUUID(), + }), + ).toEqual([]); + + await db.sql` + create function public.fail_event_destination() returns trigger language plpgsql as $$ + begin + if new.source_event_id is not null then + raise exception 'event destination insert failed'; + end if; + return new; + end; + $$ + `; + await db.sql` + create trigger fail_event_destination + before insert on pgconductor._private_executions + for each row execute function public.fail_event_destination() + `; + await expect( + db.client.dispatchCustomEvents({ eventIds: [eventId], orchestratorId: owner }), + ).rejects.toThrow("event destination insert failed"); + const [rolledBack] = await db.sql<{ destinations: string; dispatched_at: Date | null }[]>` + select + (select count(*)::text from pgconductor._private_executions where source_event_id = ${eventId}::uuid) as destinations, + (select dispatched_at from pgconductor._private_custom_events where id = ${eventId}::uuid) as dispatched_at + `; + expect(rolledBack).toEqual({ destinations: "0", dispatched_at: null }); + await db.sql`drop trigger fail_event_destination on pgconductor._private_executions`; + await db.sql`drop function public.fail_event_destination()`; + expect( + await db.client.dispatchCustomEvents({ eventIds: [eventId], orchestratorId: owner }), + ).toEqual([eventId]); + }); + + test("event destinations settle independently from their source", async () => { + const db = await database(); + await registerSubscriptions(db, [ + { taskKey: "pipeline.failure", eventKey: "pipeline.failure", maxAttempts: 1 }, + ]); + const eventId = await db.client.emitEvent({ eventKey: "pipeline.failure", payload: {} }); + const sourceOwner = await claimEvent(db, eventId); + await db.client.dispatchCustomEvents({ eventIds: [eventId], orchestratorId: sourceOwner }); + const destinationOwner = crypto.randomUUID(); + const [destination] = await db.client.getExecutions({ + orchestratorId: destinationOwner, + queueName: "default", + batchSize: 1, + filterTaskKeys: [], + }); + expect(destination).toBeDefined(); + if (!destination) throw new Error("destination was not claimed"); + await db.client.returnExecutions({ + count: 1, + orchestratorId: destinationOwner, + completed: [], + failed: [ + { + execution_id: destination.id, + orchestrator_id: destinationOwner, + queue: destination.queue, + task_key: destination.task_key, + status: "failed", + error: "destination failed", + }, + ], + released: [], + invokeChild: [], + taskKeys: new Set([destination.task_key]), + }); + const [source] = await db.sql<{ failed_at: Date | null; locked_by: string | null }[]>` + select failed_at, locked_by + from pgconductor._private_executions + where id = ${eventId}::uuid and queue = ${INTERNAL_QUEUE} + `; + expect(source?.failed_at).toBeNull(); + expect(source?.locked_by).toBe(sourceOwner); + const [failedDestination] = await db.sql< + { failed_at: Date | null; last_error: string | null }[] + >` + select failed_at, last_error + from pgconductor._private_executions + where id = ${destination.id}::uuid and queue = 'default' + `; + expect(failedDestination?.failed_at).not.toBeNull(); + expect(failedDestination?.last_error).toBe("destination failed"); + }); + + test("serializes registrations that replace subscriptions for the same queue", async () => { + const db = await database(); + const blockerSql = postgres(db.url, { max: 1 }); + const registrationSql = postgres(db.url, { max: 1 }); + const client = new DatabaseClient({ sql: registrationSql, logger: new DefaultLogger() }); + let registration: Promise | undefined; + try { + await blockerSql.begin(async (transaction) => { + await transaction` + select 1 + from pgconductor._private_queues + where name = 'default' + for update + `; + registration = client.registerWorker({ + queueName: "default", + taskSpecs: [{ key: "pipeline.serialized", queue: "default", maxAttempts: 3 }], + cronSchedules: [], + eventSubscriptions: [ + { + task_key: "pipeline.serialized", + event_key: "pipeline.serialized", + payload_fields: null, + filter: null, + }, + ], + }); + await waitForCondition(async () => { + const [waiting] = await db.sql<{ exists: boolean }[]>` + select exists( + select 1 + from pg_stat_activity + where datname = current_database() + and cardinality(pg_blocking_pids(pid)) > 0 + ) as exists + `; + return waiting?.exists === true; + }); + }); + if (!registration) throw new Error("registration did not start"); + await registration; + } finally { + await Promise.all([blockerSql.end(), registrationSql.end()]); + } + + const [count] = await db.sql<{ count: number }[]>` + select count(*)::integer as count + from pgconductor._private_custom_event_subscriptions + where queue = 'default' and event_key = 'pipeline.serialized' + `; + expect(count?.count).toBe(1); + }, 30_000); + + test("coordinates concurrent dispatch claims and drains recursively emitted fanout", async () => { + const db = await database(); + await registerSubscriptions(db, [{ taskKey: "pipeline.once", eventKey: "pipeline.once" }]); + const eventId = await db.client.emitEvent({ eventKey: "pipeline.once", payload: {} }); + const sql = postgres(db.url, { max: 2 }); + const clients = [ + new DatabaseClient({ sql, logger: new DefaultLogger() }), + new DatabaseClient({ sql, logger: new DefaultLogger() }), + ]; + try { + const owners = [crypto.randomUUID(), crypto.randomUUID()]; + const claims = await Promise.all( + clients.map((client, index) => + client.getExecutions({ + orchestratorId: owners[index] || "", + queueName: INTERNAL_QUEUE, + batchSize: 1, + filterTaskKeys: [], + }), + ), + ); + expect(claims.flat()).toHaveLength(1); + const winner = claims[0]?.length ? 0 : 1; + expect( + await clients[winner]?.dispatchCustomEvents({ + eventIds: [eventId], + orchestratorId: owners[winner] || "", + }), + ).toEqual([eventId]); + } finally { + await sql.end(); + } + + const event = defineEvent({ + name: "pipeline.drain-fanout", + payload: z.object({ value: z.string() }), + }); + const sourceDefinition = defineTask({ name: "pipeline.source", payload: z.object({}) }); + const destinationDefinition = defineTask({ + name: "pipeline.destination", + queue: "destination", + payload: z.object({}), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([sourceDefinition, destinationDefinition]), + events: EventSchemas.fromSchema([event]), + context: {}, + }); + const received: string[] = []; + const source = conductor.createTask( + { name: "pipeline.source" }, + { invocable: true }, + async () => { + await conductor.emit("pipeline.drain-fanout", { value: "done" }); + }, + ); + const destination = conductor.createTask( + { name: "pipeline.destination", queue: "destination" }, + { event: "pipeline.drain-fanout" }, + async (receivedEvent) => { + received.push(receivedEvent.payload.value); + }, + ); + await conductor.invoke({ name: "pipeline.source" }, {}); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [source], + workers: [conductor.createWorker({ queue: "destination", tasks: [destination] })], + defaultWorker: { pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.drain(); + expect(received).toEqual(["done"]); + }); + + test("runs global event maintenance for named-queue-only orchestrators", async () => { + const db = await database(); + await db.client.setFakeTime({ date: new Date(Date.now() - 10 * 24 * 60 * 60 * 1000) }); + const oldEventId = await db.client.emitEvent({ eventKey: "pipeline.old", payload: {} }); + await db.sql` + update pgconductor._private_executions + set failed_at = pgconductor._private_current_time() + where id = ${oldEventId}::uuid and queue = ${INTERNAL_QUEUE} + `; + await db.client.clearFakeTime(); + const definition = defineTask({ + name: "pipeline.named-only", + queue: "pipeline.named", + payload: z.object({}), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const task = conductor.createTask( + { name: "pipeline.named-only", queue: "pipeline.named" }, + { invocable: true }, + async () => {}, + ); + const orchestrator = Orchestrator.create({ + conductor, + workers: [conductor.createWorker({ queue: "pipeline.named", tasks: [task] })], + }); + await orchestrator.start(); + try { + const [maintenance] = await db.sql<{ task: boolean; schedule: boolean }[]>` + select + exists( + select 1 from pgconductor._private_tasks + where queue = ${INTERNAL_QUEUE} and key = 'pgconductor.maintenance' + ) as task, + exists( + select 1 from pgconductor._private_executions + where queue = ${INTERNAL_QUEUE} + and task_key = 'pgconductor.maintenance' + and cron_expression is not null + ) as schedule + `; + expect(maintenance).toEqual({ task: true, schedule: true }); + await db.client.invoke({ + task_key: "pgconductor.maintenance", + queue: INTERNAL_QUEUE, + payload: {}, + }); + await waitForCondition(async () => { + const [event] = await db.sql<{ exists: boolean }[]>` + select exists( + select 1 from pgconductor._private_custom_events where id = ${oldEventId}::uuid + ) as exists + `; + return event?.exists === false; + }); + } finally { + await orchestrator.stop(); + } + }); + + test("rejects malformed filters and the reserved internal queue", async () => { + const db = await database(); + const event = defineEvent({ + name: "pipeline.validation", + payload: z.object({ status: z.string(), metadata: z.object({ source: z.string() }) }), + filterable: ["status"], + }); + const definition = defineTask({ name: "pipeline.validation-task", payload: z.object({}) }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + events: EventSchemas.fromSchema([event]), + context: {}, + }); + expect(() => + conductor.createTask( + { name: "pipeline.validation-task" }, + { event: "pipeline.validation", filter: { status: [{ source: "api" }] } } as any, + async () => {}, + ), + ).toThrow(/scalar values/); + expect(() => + conductor.createWorker({ + queue: INTERNAL_QUEUE, + tasks: [] as never, + }), + ).toThrow(/reserved for internal use/); + }); +}); diff --git a/packages/pgconductor-js/tests/integration/event-triggers.test.ts b/packages/pgconductor-js/tests/integration/event-triggers.test.ts index 2975e54..c757618 100644 --- a/packages/pgconductor-js/tests/integration/event-triggers.test.ts +++ b/packages/pgconductor-js/tests/integration/event-triggers.test.ts @@ -4,10 +4,10 @@ import { Conductor } from "../../src/conductor"; import { Orchestrator } from "../../src/orchestrator"; import { defineTask } from "../../src/task-definition"; import { defineEvent } from "../../src/event-definition"; -import { TaskSchemas, EventSchemas, DatabaseSchema } from "../../src/schemas"; +import { TaskSchemas, EventSchemas } from "../../src/schemas"; import { TestDatabasePool } from "../fixtures/test-database"; import type { TestDatabase } from "../fixtures/test-database"; -import type { Database } from "../database.types"; +import { waitForCondition } from "../test-utils"; describe("Event Triggers - Custom Events", () => { let pool: TestDatabasePool; @@ -146,6 +146,7 @@ describe("Event Triggers - Custom Events", () => { const orderPlaced = defineEvent({ name: "order.placed", payload: z.object({ orderId: z.string(), total: z.number() }), + filterable: ["total"], }); const taskDef = defineTask({ @@ -167,7 +168,7 @@ describe("Event Triggers - Custom Events", () => { const task = conductor.createTask( { name: "on-large-order" }, - { event: "order.placed", when: "(new.payload->>'total')::numeric > 1000" }, + { event: "order.placed", filter: { total: [1500] } }, taskFn, ); @@ -255,347 +256,81 @@ describe("Event Triggers - Custom Events", () => { await orchestrator.stop(); }, 30000); -}); - -describe("Event Triggers - Database CDC", () => { - let pool: TestDatabasePool; - const databases: TestDatabase[] = []; - - beforeAll(async () => { - pool = await TestDatabasePool.create(); - }, 60000); - - afterEach(async () => { - await Promise.all(databases.map((db) => db.destroy())); - databases.length = 0; - }); - afterAll(async () => { - await pool?.destroy(); - }); - - test("task triggered on INSERT", async () => { + test("user-owned database triggers can emit transactional custom events", async () => { const db = await pool.child(); databases.push(db); - // Create contact table for testing await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - name text, - active boolean default true + create table public.contact ( + id text primary key, + email text not null ) `; - - // Insert initial data so table exists before orchestrator starts - // (triggers are created during orchestrator.start(), so table must exist first) - await db.sql`insert into public.contact (email, name) values ('initial@example.com', 'Initial')`; - - const taskDef = defineTask({ - name: "on-contact-insert", - payload: z.object({}), - }); - - const taskFn = mock(async (event) => { - expect(event.name).toBe("public.contact.insert"); - expect(event.payload.tg_op).toBe("INSERT"); - expect(event.payload.old).toBeNull(); - expect(event.payload.new).toBeTruthy(); - expect(event.payload.new.email).toBe("test@example.com"); - }); - - const conductor = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - const task = conductor.createTask( - { name: "on-contact-insert" }, - { schema: "public", table: "contact", operation: "insert", columns: "id,email,name" }, - taskFn, - ); - - const orchestrator = Orchestrator.create({ - conductor, - tasks: [task], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator.start(); - - // Insert a contact (this should trigger the task) - await db.sql`insert into public.contact (email, name) values ('test@example.com', 'Test User')`; - - // Wait for task to execute - await new Promise((r) => setTimeout(r, 300)); - - expect(taskFn).toHaveBeenCalledTimes(1); - - await orchestrator.stop(); - }, 30000); - - test("task triggered on UPDATE", async () => { - const db = await pool.child(); - databases.push(db); - - // Create contact table - await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - name text, - active boolean default true - ) - `; - - // Insert initial contact - const [contact] = await db.sql<[{ id: string }]>` - insert into public.contact (email, name) - values ('test@example.com', 'Test User') - returning id - `; - - const taskDef = defineTask({ - name: "on-contact-update", - payload: z.object({}), - }); - - const taskFn = mock(async (event) => { - expect(event.name).toBe("public.contact.update"); - expect(event.payload.tg_op).toBe("UPDATE"); - expect(event.payload.old).toBeTruthy(); - expect(event.payload.new).toBeTruthy(); - expect(event.payload.old.name).toBe("Test User"); - expect(event.payload.new.name).toBe("Updated User"); - }); - - const conductor = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - const task = conductor.createTask( - { name: "on-contact-update" }, - { schema: "public", table: "contact", operation: "update", columns: "id,email,name" }, - taskFn, - ); - - const orchestrator = Orchestrator.create({ - conductor, - tasks: [task], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator.start(); - - // Update the contact - await db.sql`update public.contact set name = 'Updated User' where id = ${contact?.id}`; - - // Wait for task to execute - await new Promise((r) => setTimeout(r, 300)); - - expect(taskFn).toHaveBeenCalledTimes(1); - - await orchestrator.stop(); - }, 30000); - - test("task triggered on DELETE", async () => { - const db = await pool.child(); - databases.push(db); - - // Create contact table await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - name text, - active boolean default true - ) - `; - - // Insert initial contact - const [contact] = await db.sql<[{ id: string }]>` - insert into public.contact (email, name) - values ('test@example.com', 'Test User') - returning id + create function public.emit_contact_created() + returns trigger + language plpgsql + as $$ + begin + perform pgconductor.emit_event( + 'contact.created', + jsonb_build_object('id', new.id, 'email', new.email) + ); + return new; + end; + $$ `; - - const taskDef = defineTask({ - name: "on-contact-delete", - payload: z.object({}), - }); - - const taskFn = mock(async (event) => { - expect(event.name).toBe("public.contact.delete"); - expect(event.payload.tg_op).toBe("DELETE"); - expect(event.payload.old).toBeTruthy(); - expect(event.payload.new).toBeNull(); - expect(event.payload.old.email).toBe("test@example.com"); - }); - - const conductor = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - const task = conductor.createTask( - { name: "on-contact-delete" }, - { schema: "public", table: "contact", operation: "delete", columns: "id,email,name" }, - taskFn, - ); - - const orchestrator = Orchestrator.create({ - conductor, - tasks: [task], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator.start(); - - // Delete the contact - await db.sql`delete from public.contact where id = ${contact?.id}`; - - // Wait for task to execute - await new Promise((r) => setTimeout(r, 300)); - - expect(taskFn).toHaveBeenCalledTimes(1); - - await orchestrator.stop(); - }, 30000); - - test("database trigger with column selection", async () => { - const db = await pool.child(); - databases.push(db); - - // Create contact table for testing await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - name text, - active boolean default true - ) + create trigger emit_contact_created + after insert on public.contact + for each row execute function public.emit_contact_created() `; - const taskDef = defineTask({ - name: "on-contact-update-columns", - payload: z.object({}), - }); - - const taskFn = mock(async (event) => { - // Should only have selected columns - expect(event.payload.new.id).toBeTruthy(); - expect(event.payload.new.email).toBe("updated@example.com"); - // Non-selected columns should not exist - expect(event.payload.new.name).toBeUndefined(); + const contactCreated = defineEvent({ + name: "contact.created", + payload: z.object({ id: z.string(), email: z.string() }), }); - + const taskDef = defineTask({ name: "handle-contact-created", payload: z.object({}) }); + const taskFn = mock(async () => {}); const conductor = Conductor.create({ sql: db.sql, tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), + events: EventSchemas.fromSchema([contactCreated]), context: {}, }); - const task = conductor.createTask( - { name: "on-contact-update-columns" }, - { schema: "public", table: "contact", operation: "update", columns: "id,email" }, + { name: "handle-contact-created" }, + { event: "contact.created" }, taskFn, ); - const orchestrator = Orchestrator.create({ conductor, tasks: [task], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, + defaultWorker: { pollIntervalMs: 10, flushIntervalMs: 10 }, }); await orchestrator.start(); + await db.sql`insert into public.contact (id, email) values ('committed', 'ok@example.com')`; + await waitForCondition(() => taskFn.mock.calls.length === 1); + + await expect( + db.sql.begin(async (transaction) => { + await transaction` + insert into public.contact (id, email) + values ('rolled-back', 'rollback@example.com') + `; + throw new Error("roll back"); + }), + ).rejects.toThrow("roll back"); - // Insert and update a contact - const [contact] = await db.sql<[{ id: string }]>` - insert into public.contact (email, name) - values ('test@example.com', 'Test User') - returning id - `; - await db.sql`update public.contact set email = 'updated@example.com', name = 'Updated Name' where id = ${contact?.id}`; - - // Wait for task to execute - await new Promise((r) => setTimeout(r, 300)); - - expect(taskFn).toHaveBeenCalledTimes(1); - - await orchestrator.stop(); - }, 30000); - - test("database trigger with when clause filters rows", async () => { - const db = await pool.child(); - databases.push(db); - - // Create contact table for testing - await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - name text, - active boolean default true - ) + const [rolledBack] = await db.sql<{ count: number }[]>` + select count(*)::int as count + from pgconductor._private_custom_events + where event_key = 'contact.created' + and payload ->> 'id' = 'rolled-back' `; - - const taskDef = defineTask({ - name: "on-contact-active", - payload: z.object({}), - }); - - const taskFn = mock(async (event) => { - // Should only be called for active contacts - expect(event.payload.new.active).toBe(true); - }); - - const conductor = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - const task = conductor.createTask( - { name: "on-contact-active" }, - { - schema: "public", - table: "contact", - operation: "insert", - when: "NEW.active = true", - columns: "id,email,name,active", - }, - taskFn, - ); - - const orchestrator = Orchestrator.create({ - conductor, - tasks: [task], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator.start(); - - // Insert inactive contact (should not trigger) - await db.sql`insert into public.contact (email, name, active) values ('inactive@example.com', 'Inactive', false)`; - - // Insert active contact (should trigger) - await db.sql`insert into public.contact (email, name, active) values ('active@example.com', 'Active', true)`; - - // Wait for tasks to execute - await new Promise((r) => setTimeout(r, 300)); - - // Should only be called once (for active contact) - expect(taskFn).toHaveBeenCalledTimes(1); + expect(rolledBack?.count).toBe(0); await orchestrator.stop(); }, 30000); diff --git a/packages/pgconductor-js/tests/integration/maintenance-task.test.ts b/packages/pgconductor-js/tests/integration/maintenance-task.test.ts index e85fe8c..2656b06 100644 --- a/packages/pgconductor-js/tests/integration/maintenance-task.test.ts +++ b/packages/pgconductor-js/tests/integration/maintenance-task.test.ts @@ -7,6 +7,7 @@ import { TestDatabasePool } from "../fixtures/test-database"; import type { TestDatabase } from "../fixtures/test-database"; import crypto from "crypto"; import { TaskSchemas } from "../../src/schemas"; +import { createMaintenanceTask } from "../../src/maintenance-task"; function hashToJitter(str: string): number { const hash = crypto.createHash("sha256").update(str).digest(); diff --git a/packages/pgconductor-js/tests/integration/schema-manager.test.ts b/packages/pgconductor-js/tests/integration/schema-manager.test.ts index 18cb73b..151dea4 100644 --- a/packages/pgconductor-js/tests/integration/schema-manager.test.ts +++ b/packages/pgconductor-js/tests/integration/schema-manager.test.ts @@ -45,6 +45,7 @@ describe("SchemaManager", () => { expect(tableNames).toContain("schema_migrations"); expect(tableNames).toContain("_private_tasks"); expect(tableNames).toContain("_private_executions"); + expect(tableNames).not.toContain("_private_event_subscriptions"); // Calling again should report no migration needed const result2 = await schemaManager.ensureLatest(controller.signal); diff --git a/packages/pgconductor-js/tests/integration/subscription-lifecycle.test.ts b/packages/pgconductor-js/tests/integration/subscription-lifecycle.test.ts index a535272..2e48b93 100644 --- a/packages/pgconductor-js/tests/integration/subscription-lifecycle.test.ts +++ b/packages/pgconductor-js/tests/integration/subscription-lifecycle.test.ts @@ -5,20 +5,22 @@ import { defineTask } from "../../src/task-definition"; import { defineEvent } from "../../src/event-definition"; import { TaskSchemas } from "../../src/schemas"; import { EventSchemas } from "../../src/schemas"; -import { DatabaseSchema } from "../../src/schemas"; import { z } from "zod"; import { TestDatabasePool, TestDatabase } from "../fixtures/test-database"; -import type { Database } from "../database.types"; +import { waitForCondition } from "../test-utils"; describe("Event Subscription Lifecycle", () => { let pool: TestDatabasePool; const databases: TestDatabase[] = []; + const orchestrators: Orchestrator[] = []; beforeAll(async () => { pool = await TestDatabasePool.create(); }, 60000); afterEach(async () => { + await Promise.all(orchestrators.map((orchestrator) => orchestrator.stop())); + orchestrators.length = 0; await Promise.all(databases.map((db) => db.destroy())); databases.length = 0; }); @@ -27,7 +29,7 @@ describe("Event Subscription Lifecycle", () => { await pool?.destroy(); }); - test("custom event triggers are created on orchestrator start", async () => { + test("custom event subscriptions are persisted and processed asynchronously", async () => { const db = await pool.child(); databases.push(db); @@ -48,10 +50,11 @@ describe("Event Subscription Lifecycle", () => { context: {}, }); + const taskFn = mock(async () => {}); const task = conductor.createTask( { name: "on-user-created" }, { event: "user.created" }, - mock(async () => {}), + taskFn, ); const orchestrator = Orchestrator.create({ @@ -59,13 +62,14 @@ describe("Event Subscription Lifecycle", () => { tasks: [task], defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, }); + orchestrators.push(orchestrator); await orchestrator.start(); - // Check subscription was created - const [sub] = await db.sql<[{ event_key: string; task_key: string }]>` - select event_key, task_key - from pgconductor._private_event_subscriptions + // Check the persistent subscription was created + const [sub] = await db.sql<[{ id: string; event_key: string; task_key: string }]>` + select id, event_key, task_key + from pgconductor._private_custom_event_subscriptions where event_key = 'user.created' `; @@ -73,94 +77,39 @@ describe("Event Subscription Lifecycle", () => { expect(sub.event_key).toBe("user.created"); expect(sub.task_key).toBe("on-user-created"); - // Check trigger function was created - const [trigger] = await db.sql<[{ exists: boolean }]>` - select exists( - select 1 - from pg_trigger - where tgname = 'pgconductor_custom_event' - and tgrelid = 'pgconductor._private_custom_events'::regclass - ) - `; - - expect(trigger.exists).toBe(true); - - await orchestrator.stop(); - }, 30000); - - test("database triggers are created on orchestrator start", async () => { - const db = await pool.child(); - databases.push(db); - - // Create contact table - await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - name text - ) + const [compiledFilter] = await db.sql< + [{ filter: Record; field_count: number; predicate_count: string }] + >` + select subscription.filter, subscription.field_count, count(predicate.id)::text as predicate_count + from pgconductor._private_custom_event_subscriptions subscription + left join pgconductor._private_custom_event_predicates predicate + on predicate.subscription_id = subscription.id + where subscription.id = ${sub.id} + group by subscription.id `; - - const taskDef = defineTask({ - name: "on-contact-insert", - payload: z.object({}), - }); - - const conductor = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, + expect(compiledFilter).toEqual({ filter: {}, field_count: 0, predicate_count: "0" }); + + const eventId = await conductor.emit("user.created", { userId: "user-123" }); + await waitForCondition(() => taskFn.mock.calls.length === 1); + await waitForCondition(async () => { + const [source] = await db.sql<{ exists: boolean }[]>` + select exists( + select 1 from pgconductor._private_executions + where id = ${eventId}::uuid and queue = 'pgconductor.internal' + ) as exists + `; + return !source?.exists; }); - - const task = conductor.createTask( - { name: "on-contact-insert" }, - { schema: "public", table: "contact", operation: "insert", columns: "id,email,name" }, - mock(async () => {}), - ); - - const orchestrator = Orchestrator.create({ - conductor, - tasks: [task], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator.start(); - - // Check subscription was created - const [sub] = await db.sql<[{ schema_name: string; table_name: string; operation: string }]>` - select schema_name, table_name, operation::text - from pgconductor._private_event_subscriptions - where schema_name = 'public' and table_name = 'contact' and operation = 'insert' - `; - - expect(sub).toBeTruthy(); - expect(sub.schema_name).toBe("public"); - expect(sub.table_name).toBe("contact"); - expect(sub.operation).toBe("insert"); - - // Check trigger was created on table - const [trigger] = await db.sql<[{ exists: boolean }]>` - select exists( - select 1 - from pg_trigger - where tgname = 'pgconductor_event_insert' - and tgrelid = 'public.contact'::regclass - ) - `; - - expect(trigger.exists).toBe(true); - - await orchestrator.stop(); }, 30000); - test("stopping orchestrator does not remove triggers", async () => { + test("custom event subscriptions and compiled filters persist after stop", async () => { const db = await pool.child(); databases.push(db); const userCreated = defineEvent({ name: "user.created.persistent", payload: z.object({ userId: z.string() }), + filterable: ["userId"], }); const taskDef = defineTask({ @@ -177,7 +126,10 @@ describe("Event Subscription Lifecycle", () => { const task = conductor.createTask( { name: "on-user-persistent" }, - { event: "user.created.persistent" }, + { + event: "user.created.persistent", + filter: { userId: ["user-123", "user-456", "user-123"] }, + }, mock(async () => {}), ); @@ -186,45 +138,69 @@ describe("Event Subscription Lifecycle", () => { tasks: [task], defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, }); + orchestrators.push(orchestrator); await orchestrator.start(); - // Verify trigger exists - const [beforeStop] = await db.sql<[{ exists: boolean }]>` - select exists( + const [beforeStop] = await db.sql<[{ id: string; exists: boolean }]>` + select subscription.id, exists( select 1 - from pg_trigger - where tgname = 'pgconductor_custom_event' - and tgrelid = 'pgconductor._private_custom_events'::regclass - ) + from pgconductor._private_custom_event_predicates predicate + where predicate.subscription_id = subscription.id + ) as exists + from pgconductor._private_custom_event_subscriptions subscription + where event_key = 'user.created.persistent' `; + expect(beforeStop).toBeTruthy(); expect(beforeStop.exists).toBe(true); + const [invariants] = await db.sql< + { + predicates: number; + field_count: number; + events_match: boolean; + has_expected_predicate: boolean; + }[] + >` + select + count(predicate.id)::integer as predicates, + subscription.field_count, + bool_and(predicate.event_key = subscription.event_key) as events_match, + bool_or( + predicate.field_name = 'userId' + and predicate.value = '"user-123"'::jsonb + ) as has_expected_predicate + from pgconductor._private_custom_event_subscriptions subscription + join pgconductor._private_custom_event_predicates predicate + on predicate.subscription_id = subscription.id + where subscription.id = ${beforeStop.id} + group by subscription.id + `; + expect(invariants).toEqual({ + predicates: 2, + field_count: 1, + events_match: true, + has_expected_predicate: true, + }); + await orchestrator.stop(); - // Verify trigger still exists after stop - const [afterStop] = await db.sql<[{ exists: boolean }]>` + const [afterStop] = await db.sql<[{ exists: boolean; filter_count: string }]>` select exists( - select 1 - from pg_trigger - where tgname = 'pgconductor_custom_event' - and tgrelid = 'pgconductor._private_custom_events'::regclass - ) + select 1 from pgconductor._private_custom_event_subscriptions + where id = ${beforeStop.id} + ) as exists, + ( + select count(*)::text + from pgconductor._private_custom_event_predicates predicate + where predicate.subscription_id = ${beforeStop.id} + ) as filter_count `; expect(afterStop.exists).toBe(true); - - // Verify subscription still exists - const [sub] = await db.sql<[{ exists: boolean }]>` - select exists( - select 1 - from pgconductor._private_event_subscriptions - where event_key = 'user.created.persistent' - ) - `; - expect(sub.exists).toBe(true); + expect(afterStop.filter_count).toBe("2"); }, 30000); - test("triggers are recreated when subscriptions change", async () => { + test("custom event subscriptions are replaced when their configuration changes", async () => { const db = await pool.child(); databases.push(db); @@ -262,13 +238,14 @@ describe("Event Subscription Lifecycle", () => { tasks: [task1], defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, }); + orchestrators.push(orchestrator1); await orchestrator1.start(); // Check subscription without field selection const [sub1] = await db.sql<[{ payload_fields: string[] | null }]>` select payload_fields - from pgconductor._private_event_subscriptions + from pgconductor._private_custom_event_subscriptions where event_key = 'user.created.change' `; expect(sub1.payload_fields).toBeNull(); @@ -294,13 +271,14 @@ describe("Event Subscription Lifecycle", () => { ], defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, }); + orchestrators.push(orchestrator2); await orchestrator2.start(); // Check subscription was updated with field selection const [sub2] = await db.sql<[{ payload_fields: string[]; task_key: string }]>` select payload_fields, task_key - from pgconductor._private_event_subscriptions + from pgconductor._private_custom_event_subscriptions where event_key = 'user.created.change' `; expect(sub2.payload_fields).toEqual(["userId"]); @@ -352,13 +330,14 @@ describe("Event Subscription Lifecycle", () => { tasks: [task1, task2], defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, }); + orchestrators.push(orchestrator); await orchestrator.start(); // Check both subscriptions exist const subs = await db.sql<{ task_key: string }[]>` select task_key - from pgconductor._private_event_subscriptions + from pgconductor._private_custom_event_subscriptions where event_key = 'user.created.multi' order by task_key `; @@ -371,93 +350,4 @@ describe("Event Subscription Lifecycle", () => { await orchestrator.stop(); }, 30000); - - test("when clause changes trigger recreation", async () => { - const db = await pool.child(); - databases.push(db); - - // Create contact table - await db.sql` - create table if not exists public.contact ( - id text primary key default gen_random_uuid()::text, - email text, - active boolean default true - ) - `; - - const taskDef = defineTask({ - name: "on-contact-conditional", - payload: z.object({}), - }); - - // First orchestrator without when clause - const conductor1 = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - const task1 = conductor1.createTask( - { name: "on-contact-conditional" }, - { schema: "public", table: "contact", operation: "insert", columns: "id,email,active" }, - mock(async () => {}), - ); - - const orchestrator1 = Orchestrator.create({ - conductor: conductor1, - tasks: [task1], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator1.start(); - - // Check subscription without when clause - const [sub1] = await db.sql<[{ when_clause: string | null }]>` - select when_clause - from pgconductor._private_event_subscriptions - where schema_name = 'public' and table_name = 'contact' - `; - expect(sub1.when_clause).toBeNull(); - - await orchestrator1.stop(); - - // Second orchestrator with when clause - const conductor2 = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - const task2 = conductor2.createTask( - { name: "on-contact-conditional" }, - { - schema: "public", - table: "contact", - operation: "insert", - when: "NEW.active = true", - columns: "id,email,active", - }, - mock(async () => {}), - ); - - const orchestrator2 = Orchestrator.create({ - conductor: conductor2, - tasks: [task2], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, - }); - - await orchestrator2.start(); - - // Check subscription was updated with when clause - const [sub2] = await db.sql<[{ when_clause: string }]>` - select when_clause - from pgconductor._private_event_subscriptions - where schema_name = 'public' and table_name = 'contact' - `; - expect(sub2.when_clause).toBe("NEW.active = true"); - - await orchestrator2.stop(); - }, 30000); }); diff --git a/packages/pgconductor-js/tests/mocks/database-client.mock.ts b/packages/pgconductor-js/tests/mocks/database-client.mock.ts index b799b5f..a546d15 100644 --- a/packages/pgconductor-js/tests/mocks/database-client.mock.ts +++ b/packages/pgconductor-js/tests/mocks/database-client.mock.ts @@ -19,6 +19,7 @@ export class MockDatabaseClient implements IDatabaseClient { getExecutions = mock(async () => []); returnExecutions = mock(async (_results) => {}); removeExecutions = mock(async () => false); + removeCustomEvents = mock(async (..._args: any[]) => false); registerWorker = mock(async () => {}); scheduleCronExecution = mock(async () => "mock-cron-id"); unscheduleCronExecution = mock(async () => {}); @@ -33,9 +34,8 @@ export class MockDatabaseClient implements IDatabaseClient { getDatabaseTime = mock(async () => new Date()); setFakeTime = mock(async () => {}); clearFakeTime = mock(async () => {}); - subscribeEvent = mock(async () => "mock-subscription-id"); - subscribeDbChange = mock(async () => "mock-subscription-id"); emitEvent = mock(async () => "mock-event-id"); + dispatchCustomEvents = mock(async ({ eventIds }: { eventIds: string[] }) => eventIds); constructor(overrides: Partial = {}) { Object.assign(this, overrides); diff --git a/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts b/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts index 73f11ce..d2ef526 100644 --- a/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts +++ b/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts @@ -16,6 +16,7 @@ import type { CountActiveOrchestratorsBelowArgs, GetExecutionsArgs, RemoveExecutionsArgs, + RemoveCustomEventsArgs, RegisterWorkerArgs, ScheduleCronExecutionArgs, UnscheduleCronExecutionArgs, @@ -61,6 +62,8 @@ interface StoredExecution { created_at: Date; updated_at: Date; failed_at: Date | null; + source_event_id: string | null; + event_subscription_id: string | null; dead_letter_source_execution_id: string | null; dead_letter_source_queue: string | null; dead_letter_source_task_key: string | null; @@ -107,15 +110,11 @@ interface StoredOrchestrator { interface StoredEventSubscription { id: string; - execution_id: string; - step_key: string; - source: "event" | "db"; - event_key?: string; - schema_name?: string; - table_name?: string; - operation?: string; - columns?: string[]; - timeout_at: Date | null; + task_key: string; + queue: string; + event_key: string; + payload_fields: string[] | null; + filter: Record | null; } interface SignalData { @@ -133,7 +132,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { private cronSchedules = new Map(); private orchestrators = new Map(); private eventSubscriptions = new Map(); - private eventPartitions = new Set(); + private dispatchedEvents = new Set(); private currentTime: Date; private migrationNumber = -1; private idCounter = 0; @@ -309,6 +308,23 @@ export class InMemoryDatabaseClient implements IDatabaseClient { this.tasks.set(this.taskId(taskSpec.key, task.queue), task); } + // Registration is authoritative for this queue, just like PostgreSQL. + for (const id of [...this.eventSubscriptions.keys()]) { + if (this.eventSubscriptions.get(id)?.queue === args.queueName) + this.eventSubscriptions.delete(id); + } + for (const spec of args.eventSubscriptions || []) { + const id = this.generateId(); + this.eventSubscriptions.set(id, { + id, + task_key: spec.task_key, + queue: args.queueName, + event_key: spec.event_key, + payload_fields: spec.payload_fields, + filter: spec.filter as Record | null, + }); + } + // Register cron schedules (ExecutionSpec[]) for (const cronSpec of args.cronSchedules || []) { if (cronSpec.cron_expression) { @@ -416,6 +432,8 @@ export class InMemoryDatabaseClient implements IDatabaseClient { dedupe_key: exec.dedupe_key || undefined, cron_expression: exec.cron_expression || undefined, group: exec.group, + source_event_id: exec.source_event_id, + event_subscription_id: exec.event_subscription_id, dead_letter_source_execution_id: exec.dead_letter_source_execution_id, dead_letter_source_queue: exec.dead_letter_source_queue, dead_letter_source_task_key: exec.dead_letter_source_task_key, @@ -642,30 +660,6 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // exec.orchestrator_id = null; // break; // } - - // case "wait_for_db_event": { - // // Create subscription - // const subscriptionId = this.generateId(); - // this.eventSubscriptions.set(subscriptionId, { - // id: subscriptionId, - // execution_id: exec.id, - // step_key: result.step_key, - // source: "db", - // schema_name: result.schema_name, - // table_name: result.table_name, - // operation: result.operation, - // columns: result.columns, - // timeout_at: - // result.timeout_ms === "infinity" - // ? new Date(8640000000000000) - // : new Date(now.getTime() + result.timeout_ms), - // }); - - // exec.state = "pending"; - // exec.run_at = new Date(8640000000000000); // Wait indefinitely - // exec.orchestrator_id = null; - // break; - // } } exec.updated_at = now; @@ -673,12 +667,69 @@ export class InMemoryDatabaseClient implements IDatabaseClient { } async removeExecutions( - _args: RemoveExecutionsArgs, + args: RemoveExecutionsArgs, _opts?: { signal?: AbortSignal }, ): Promise { - // In the real implementation, this removes old completed/failed executions - // For the in-memory client, we'll just return true (could be enhanced later) - return true; + const now = this.getInternalTime(); + const candidates = Array.from(this.executions.values()) + .filter((exec) => { + if ( + exec.queue !== args.queueName || + (exec.state !== "completed" && exec.state !== "failed") + ) { + return false; + } + const task = this.tasks.get(this.taskId(exec.task_key, exec.queue)); + if (!task) return false; + const retentionDays = + exec.state === "completed" ? task.remove_on_complete_days : task.remove_on_fail_days; + if (retentionDays == null || retentionDays <= 0) return false; + const terminalAt = exec.state === "completed" ? exec.updated_at : exec.failed_at; + return ( + terminalAt != null && terminalAt.getTime() < now.getTime() - retentionDays * 86400000 + ); + }) + .slice(0, args.batchSize); + + for (const exec of candidates) { + this.executions.delete(exec.id); + this.steps.delete(exec.id); + } + return candidates.length >= args.batchSize; + } + + async removeCustomEvents( + before: Date, + batchSize: number, + _opts?: { signal?: AbortSignal }, + ): Promise; + async removeCustomEvents( + args: RemoveCustomEventsArgs, + _opts?: { signal?: AbortSignal }, + ): Promise; + async removeCustomEvents( + beforeOrArgs: Date | RemoveCustomEventsArgs, + batchSizeOrOpts?: number | { signal?: AbortSignal }, + _opts?: { signal?: AbortSignal }, + ): Promise { + const before = beforeOrArgs instanceof Date ? beforeOrArgs : beforeOrArgs.before; + const batchSize = + beforeOrArgs instanceof Date ? (batchSizeOrOpts as number) : beforeOrArgs.batchSize; + const candidates = Array.from(this.executions.values()) + .filter( + (exec) => + exec.queue === "pgconductor.internal" && + exec.task_key === "pgconductor.event-dispatch" && + exec.created_at < before && + (exec.state === "completed" || exec.state === "failed" || exec.cancelled), + ) + .slice(0, batchSize); + + for (const exec of candidates) { + this.executions.delete(exec.id); + this.steps.delete(exec.id); + } + return candidates.length >= batchSize; } async invoke(spec: ExecutionSpec, _opts?: { signal?: AbortSignal }): Promise { @@ -740,6 +791,8 @@ export class InMemoryDatabaseClient implements IDatabaseClient { created_at: now, updated_at: now, failed_at: null, + source_event_id: null, + event_subscription_id: null, dead_letter_source_execution_id: null, dead_letter_source_queue: null, dead_letter_source_task_key: null, @@ -1095,21 +1148,76 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // return eventId; // } // - // // Methods referenced in mock but not in main interface - // async subscribeEvent(): Promise { - // return this.generateId(); - // } - // - // async subscribeDbChange(): Promise { - // return this.generateId(); - // } async invokeChild(): Promise { return this.generateId(); } - async emitEvent(): Promise { - return this.generateId(); + async emitEvent(args: { eventKey: string; payload?: unknown }): Promise { + const id = await this.invoke({ + task_key: "pgconductor.event-dispatch", + queue: "pgconductor.internal", + payload: { + event_key: args.eventKey, + payload: args.payload && typeof args.payload === "object" ? (args.payload as Payload) : {}, + }, + }); + if (!id) throw new Error("Failed to persist event execution"); + return id; + } + + async dispatchCustomEvents(args: { + eventIds: string[]; + orchestratorId: string; + }): Promise { + const dispatched: string[] = []; + for (const eventId of args.eventIds) { + const source = this.executions.get(eventId); + if ( + !source || + source.queue !== "pgconductor.internal" || + source.task_key !== "pgconductor.event-dispatch" || + source.orchestrator_id !== args.orchestratorId || + source.cancelled + ) { + continue; + } + if (this.dispatchedEvents.has(eventId)) { + dispatched.push(eventId); + continue; + } + const eventKey = String(source.payload.event_key); + const payload = source.payload.payload as Payload; + for (const subscription of this.eventSubscriptions.values()) { + if (subscription.event_key !== eventKey) continue; + const matches = Object.entries(subscription.filter || {}).every(([field, values]) => + values.some((value) => Object.is(value, payload[field])), + ); + if (!matches) continue; + const destinationPayload = subscription.payload_fields + ? (Object.fromEntries( + subscription.payload_fields + .filter((field) => Object.prototype.hasOwnProperty.call(payload, field)) + .map((field) => [field, payload[field]]), + ) as Payload) + : structuredClone(payload); + const destinationId = await this.invoke({ + task_key: subscription.task_key, + queue: subscription.queue, + payload: { event: eventKey, payload: destinationPayload }, + }); + if (destinationId) { + const destination = this.executions.get(destinationId); + if (destination) { + destination.source_event_id = eventId; + destination.event_subscription_id = subscription.id; + } + } + } + this.dispatchedEvents.add(eventId); + dispatched.push(eventId); + } + return dispatched; } // ============================================================================ @@ -1155,6 +1263,8 @@ export class InMemoryDatabaseClient implements IDatabaseClient { created_at: now, updated_at: now, failed_at: null, + source_event_id: null, + event_subscription_id: null, dead_letter_source_execution_id: exec.id, dead_letter_source_queue: exec.queue, dead_letter_source_task_key: exec.task_key, @@ -1240,7 +1350,6 @@ export class InMemoryDatabaseClient implements IDatabaseClient { this.cronSchedules.clear(); this.orchestrators.clear(); this.eventSubscriptions.clear(); - this.eventPartitions.clear(); this.idCounter = 0; } diff --git a/packages/pgconductor-js/tests/unit/emit-types.test.ts b/packages/pgconductor-js/tests/unit/emit-types.test.ts index 09a6244..e92eb72 100644 --- a/packages/pgconductor-js/tests/unit/emit-types.test.ts +++ b/packages/pgconductor-js/tests/unit/emit-types.test.ts @@ -41,11 +41,13 @@ describe("emit method types", () => { Promise >(); - // @ts-expect-error - wrong payload type - conductor.emit("user.created", { orderId: 123 }); + if (false) { + // @ts-expect-error - wrong payload type + conductor.emit("user.created", { orderId: 123 }); - // @ts-expect-error - non-existent event - conductor.emit("non.existent", {}); + // @ts-expect-error - non-existent event + conductor.emit("non.existent", {}); + } }); test("ctx.emit accepts typed event payload", () => { diff --git a/packages/pgconductor-js/tests/unit/event-pipeline-types.test.ts b/packages/pgconductor-js/tests/unit/event-pipeline-types.test.ts new file mode 100644 index 0000000..e722ba2 --- /dev/null +++ b/packages/pgconductor-js/tests/unit/event-pipeline-types.test.ts @@ -0,0 +1,25 @@ +import { describe, expectTypeOf, test } from "bun:test"; +import { z } from "zod"; +import { defineEvent, type FilterForEvent } from "../../src/event-definition"; + +describe("event filter types", () => { + test("omitting filterable fields permits no filters", () => { + const event = defineEvent({ + name: "account.created", + payload: z.object({ accountId: z.string() }), + }); + expectTypeOf>().toEqualTypeOf<{}>(); + }); + + test("infer allowed fields and values from the payload", () => { + const event = defineEvent({ + name: "order.changed", + payload: z.object({ status: z.enum(["pending", "paid"]), accountId: z.string() }), + filterable: ["status"], + }); + + expectTypeOf>().toEqualTypeOf<{ + readonly status?: readonly ("pending" | "paid")[]; + }>(); + }); +}); diff --git a/packages/pgconductor-js/tests/unit/event-trigger-types.test.ts b/packages/pgconductor-js/tests/unit/event-trigger-types.test.ts index 1b5f5e6..6ac4e8c 100644 --- a/packages/pgconductor-js/tests/unit/event-trigger-types.test.ts +++ b/packages/pgconductor-js/tests/unit/event-trigger-types.test.ts @@ -2,10 +2,9 @@ import { test, expect, describe } from "bun:test"; import { expectTypeOf } from "expect-type"; import { Conductor } from "../../src/conductor"; import { defineTask } from "../../src/task-definition"; -import { defineEvent } from "../../src/event-definition"; -import { TaskSchemas, EventSchemas, DatabaseSchema } from "../../src/schemas"; +import { defineEvent, type DefineEvent } from "../../src/event-definition"; +import { TaskSchemas, EventSchemas } from "../../src/schemas"; import { z } from "zod"; -import type { Database } from "../database.types"; describe("event triggers", () => { test("task with custom event trigger receives typed event", () => { @@ -36,41 +35,6 @@ describe("event triggers", () => { }); }); - test("task with database event trigger receives typed payload", () => { - const taskDef = defineTask({ - name: "on-contact-insert", - payload: z.object({}), - }); - - const conductor = Conductor.create({ - sql: {} as any, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - // Task triggered by database insert - not invocable - conductor.createTask( - { name: "on-contact-insert" }, - { - schema: "public", - table: "contact", - operation: "insert", - columns: "id,email,first_name", - }, - async (event) => { - // Event should have database event payload with schema.table.op format - expectTypeOf(event.name).toEqualTypeOf<"public.contact.insert">(); - expectTypeOf(event.payload.tg_op).toEqualTypeOf<"INSERT">(); - expectTypeOf(event.payload.old).toEqualTypeOf(); - // new should have selected contact columns - expectTypeOf(event.payload.new.id).toEqualTypeOf(); - expectTypeOf(event.payload.new.email).toEqualTypeOf(); - expectTypeOf(event.payload.new.first_name).toEqualTypeOf(); - }, - ); - }); - test("task with multiple triggers including event trigger", () => { const userCreated = defineEvent({ name: "user.created", @@ -104,44 +68,6 @@ describe("event triggers", () => { ); }); - test("task with cron and event triggers", () => { - const taskDef = defineTask({ - name: "cron-and-event", - payload: z.object({}), - }); - - const conductor = Conductor.create({ - sql: {} as any, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, - }); - - // Task with cron and database event trigger - neither is invocable - conductor.createTask( - { name: "cron-and-event" }, - [ - { cron: "0 * * * *", name: "hourly" }, - { - schema: "public", - table: "contact", - operation: "update", - columns: "id,email", - }, - ], - async (event) => { - // Event should be union of cron and database event - if (event.name === "hourly") { - // Cron event has no payload - expectTypeOf(event).toEqualTypeOf<{ name: "hourly" }>(); - } else if (event.name === "public.contact.update") { - // Database update event - expectTypeOf(event.payload.tg_op).toEqualTypeOf<"UPDATE">(); - } - }, - ); - }); - test("custom event trigger with field selection", () => { const userCreated = defineEvent({ name: "user.created", @@ -193,183 +119,154 @@ describe("event triggers", () => { ); }); - test("custom event trigger with when clause", () => { - const orderPlaced = defineEvent({ - name: "order.placed", - payload: z.object({ orderId: z.string(), total: z.number() }), - }); - - const taskDef = defineTask({ - name: "on-large-order", - payload: z.object({}), + test("custom event triggers validate event names and projected fields", () => { + const userCreated = defineEvent({ + name: "user.created", + payload: z.object({ userId: z.string(), email: z.string() }), }); - + const taskDef = defineTask({ name: "on-user-created", payload: z.object({}) }); const conductor = Conductor.create({ sql: {} as any, tasks: TaskSchemas.fromSchema([taskDef]), - events: EventSchemas.fromSchema([orderPlaced]), + events: EventSchemas.fromSchema([userCreated]), context: {}, }); - // Task with when clause - still receives full payload - conductor.createTask( - { name: "on-large-order" }, - { event: "order.placed", when: "new.payload->>'total'::numeric > 1000" }, - async (event) => { - expectTypeOf(event.payload).toEqualTypeOf<{ - orderId: string; - total: number; - }>(); - }, - ); + if (false) { + conductor.createTask( + { name: "on-user-created" }, + // @ts-expect-error The event is not in the conductor catalog. + { event: "user.typo" }, + async () => {}, + ); + conductor.createTask( + { name: "on-user-created" }, + // @ts-expect-error The selected field is not in the event payload. + { event: "user.created", fields: "userId,missing" }, + async () => {}, + ); + conductor.createTask( + { name: "on-user-created" }, + // @ts-expect-error Selected event fields must be unique. + { event: "user.created", fields: "userId,userId" }, + async () => {}, + ); + conductor.createTask( + { name: "on-user-created" }, + // @ts-expect-error Event field names use unquoted identifier syntax. + { event: "user.created", fields: '"userId"' }, + async () => {}, + ); + } + + expect(() => + conductor.createTask( + { name: "on-user-created" }, + { event: "user.typo" } as any, + async () => {}, + ), + ).toThrow('Event "user.typo" is not defined in the conductor event catalog'); + expect(() => + conductor.createTask( + { name: "on-user-created" }, + { event: "user.created", fields: "userId,userId" } as any, + async () => {}, + ), + ).toThrow('Fields for event "user.created" cannot contain duplicate names'); + expect(() => + conductor.createTask( + { name: "on-user-created" }, + { event: "user.created", fields: '"userId"' } as any, + async () => {}, + ), + ).toThrow('Fields for event "user.created" contains invalid field'); }); - test("database trigger with column selection", () => { - const taskDef = defineTask({ - name: "on-contact-update-columns", - payload: z.object({}), + test("type-only events remain valid beside runtime event definitions", () => { + type ExternalEvent = DefineEvent<{ + name: "external.received"; + payload: { externalId: string }; + filterable: ["externalId"]; + }>; + const runtimeEvent = defineEvent({ + name: "user.created", + payload: z.object({ userId: z.string() }), }); - + const taskDef = defineTask({ name: "external-task", payload: z.object({}) }); const conductor = Conductor.create({ sql: {} as any, tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), + events: EventSchemas.fromSchema([runtimeEvent]).fromUnion(), context: {}, }); - // Task with column selection - should only receive selected columns - conductor.createTask( - { name: "on-contact-update-columns" }, - { - schema: "public", - table: "contact", - operation: "update", - columns: "id,email", - }, - async (event) => { - expectTypeOf(event.name).toEqualTypeOf<"public.contact.update">(); - expectTypeOf(event.payload.tg_op).toEqualTypeOf<"UPDATE">(); - - // OLD and NEW should only have selected columns - if (event.payload.old) { - expectTypeOf(event.payload.old).toEqualTypeOf<{ - id: string; - email: string | null; - }>(); - - // @ts-expect-error - name was not selected - event.payload.old.name; - } - - if (event.payload.new) { - expectTypeOf(event.payload.new).toEqualTypeOf<{ - id: string; - email: string | null; - }>(); - - // @ts-expect-error - name was not selected - event.payload.new.name; - } - }, - ); + expect(() => + conductor.createTask( + { name: "external-task" }, + { event: "external.received", filter: { externalId: ["external-1"] } }, + async () => {}, + ), + ).not.toThrow(); }); - test("database trigger with when clause", () => { - const taskDef = defineTask({ - name: "on-contact-active", - payload: z.object({}), - }); + test("filterable event fields must contain scalar values", () => { + if (false) { + // @ts-expect-error Object-valued payload fields cannot be filterable. + defineEvent({ + name: "user.metadata", + payload: z.object({ userId: z.string(), metadata: z.object({ source: z.string() }) }), + filterable: ["metadata"], + }); + } + }); - const conductor = Conductor.create({ - sql: {} as any, - tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), - context: {}, + test("custom event trigger rejects a when clause", () => { + const orderPlaced = defineEvent({ + name: "order.placed", + payload: z.object({ orderId: z.string(), total: z.number() }), }); - // Task with when clause - still receives selected columns - conductor.createTask( - { name: "on-contact-active" }, - { - schema: "public", - table: "contact", - operation: "insert", - when: "NEW.active = true", - columns: "id,email,first_name", - }, - async (event) => { - if (event.payload.new) { - expectTypeOf(event.payload.new.id).toEqualTypeOf(); - expectTypeOf(event.payload.new.email).toEqualTypeOf(); - expectTypeOf(event.payload.new.first_name).toEqualTypeOf(); - } - }, - ); - }); - - test("database trigger DELETE operation has correct OLD/NEW types", () => { const taskDef = defineTask({ - name: "on-contact-delete", + name: "on-large-order", payload: z.object({}), }); const conductor = Conductor.create({ sql: {} as any, tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), + events: EventSchemas.fromSchema([orderPlaced]), context: {}, }); - conductor.createTask( - { name: "on-contact-delete" }, - { - schema: "public", - table: "contact", - operation: "delete", - columns: "id", - }, - async (event) => { - expectTypeOf(event.payload.tg_op).toEqualTypeOf<"DELETE">(); - // DELETE has OLD but not NEW - expectTypeOf(event.payload.new).toEqualTypeOf(); - if (event.payload.old) { - expectTypeOf(event.payload.old.id).toEqualTypeOf(); - } - }, - ); + expect(() => + conductor.createTask( + { name: "on-large-order" }, + { event: "order.placed", when: "new.payload->>'total'::numeric > 1000" }, + async (event) => { + expectTypeOf(event.payload).toEqualTypeOf<{ + orderId: string; + total: number; + }>(); + }, + ), + ).toThrow("does not support a when clause"); }); - test("database trigger UPDATE operation has both OLD and NEW", () => { - const taskDef = defineTask({ - name: "on-contact-update", - payload: z.object({}), - }); - + test("managed database trigger configuration is not exposed", () => { + const taskDef = defineTask({ name: "database-change", payload: z.object({}) }); const conductor = Conductor.create({ sql: {} as any, tasks: TaskSchemas.fromSchema([taskDef]), - database: DatabaseSchema.fromGeneratedTypes(), context: {}, + // @ts-expect-error Database schemas are not a Conductor option. + database: {}, }); conductor.createTask( - { name: "on-contact-update" }, - { - schema: "public", - table: "contact", - operation: "update", - columns: "id", - }, - async (event) => { - expectTypeOf(event.payload.tg_op).toEqualTypeOf<"UPDATE">(); - // UPDATE has both OLD and NEW - if (event.payload.old) { - expectTypeOf(event.payload.old.id).toEqualTypeOf(); - } - if (event.payload.new) { - expectTypeOf(event.payload.new.id).toEqualTypeOf(); - } - }, + { name: "database-change" }, + // @ts-expect-error Applications own database triggers and emit custom events from them. + { schema: "public", table: "contact", operation: "insert", columns: "id" }, + async () => {}, ); }); }); diff --git a/packages/pgconductor-js/tests/unit/event-trigger-validation.test.ts b/packages/pgconductor-js/tests/unit/event-trigger-validation.test.ts new file mode 100644 index 0000000..a58537d --- /dev/null +++ b/packages/pgconductor-js/tests/unit/event-trigger-validation.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { compileEventTrigger } from "../../src/event-trigger-validation"; + +describe("event trigger compilation", () => { + test("canonicalizes field order and deduplicates scalar alternatives type-sensitively", () => { + const compiled = compileEventTrigger( + { + event: "catalog.changed", + filter: { + z: [1, 1, 0], + a: ["1", 1, true, null], + }, + }, + [], + true, + ); + + expect(compiled).toEqual({ + event_key: "catalog.changed", + payload_fields: null, + filter: { + a: [null, true, 1, "1"], + z: [0, 1], + }, + }); + }); + + test("enforces index-safe UTF-8 byte limits before registration", () => { + expect(() => compileEventTrigger({ event: "e".repeat(256) }, [], true)).toThrow( + /255 UTF-8 bytes/, + ); + expect(() => + compileEventTrigger( + { event: "catalog.changed", filter: { value: ["x".repeat(1023)] } }, + [], + true, + ), + ).toThrow(/1024 UTF-8 bytes/); + }); +}); diff --git a/packages/pgconductor-js/tests/unit/task-event-types.test.ts b/packages/pgconductor-js/tests/unit/task-event-types.test.ts index ee15f44..ab53581 100644 --- a/packages/pgconductor-js/tests/unit/task-event-types.test.ts +++ b/packages/pgconductor-js/tests/unit/task-event-types.test.ts @@ -3,7 +3,7 @@ import { expectTypeOf } from "expect-type"; import { Conductor } from "../../src/conductor"; import { defineTask } from "../../src/task-definition"; // import { defineEvent } from "../../src/event-definition"; -import { TaskSchemas /*, EventSchemas, DatabaseSchema */ } from "../../src/schemas"; +import { TaskSchemas /*, EventSchemas */ } from "../../src/schemas"; import { z } from "zod"; import type { Database } from "../database.types"; @@ -242,72 +242,6 @@ describe("task event types", () => { // expect(task.name).toBe("on-user-created"); // }); // - // test.skip("createTask with only database event trigger", () => { - // const taskDef = defineTask({ - // name: "on-contact-insert", - // }); - // - // const conductor = Conductor.create({ - // sql: {} as any, - // tasks: TaskSchemas.fromSchema([taskDef]), - // database: DatabaseSchema.fromGeneratedTypes(), - // context: {}, - // }); - // - // const task = conductor.createTask( - // { name: "on-contact-insert" }, - // { schema: "public", table: "contact", operation: "insert" }, - // async (event, _ctx) => { - // // Event should be the database event - // expectTypeOf(event.name).toEqualTypeOf<"public.contact.insert">(); - // expectTypeOf(event.payload.tg_op).toEqualTypeOf<"INSERT">(); - // expectTypeOf(event.payload.old).toEqualTypeOf(); - // - // // new should have contact row type - // if (event.payload.new) { - // expectTypeOf(event.payload.new.id).toEqualTypeOf(); - // expectTypeOf(event.payload.new.first_name).toEqualTypeOf(); - // expectTypeOf(event.payload.new.email).toEqualTypeOf(); - // } - // }, - // ); - // - // expect(task.name).toBe("on-contact-insert"); - // }); - // - // test.skip("createTask with database event trigger and column selection", () => { - // const taskDef = defineTask({ - // name: "on-contact-columns", - // }); - // - // const conductor = Conductor.create({ - // sql: {} as any, - // tasks: TaskSchemas.fromSchema([taskDef]), - // database: DatabaseSchema.fromGeneratedTypes(), - // context: {}, - // }); - // - // conductor.createTask( - // { name: "on-contact-columns" }, - // { schema: "public", table: "contact", operation: "insert", columns: "id, email" }, - // async (event, _ctx) => { - // // Event should be the database event with column selection - // expectTypeOf(event.name).toEqualTypeOf<"public.contact.insert">(); - // expectTypeOf(event.payload.tg_op).toEqualTypeOf<"INSERT">(); - // expectTypeOf(event.payload.old).toEqualTypeOf(); - // - // // new should only have selected columns - // if (event.payload.new) { - // expectTypeOf(event.payload.new.id).toEqualTypeOf(); - // expectTypeOf(event.payload.new.email).toEqualTypeOf(); - // - // // @ts-expect-error - first_name not in column selection - // const _invalid = event.payload.new.first_name; - // } - // }, - // ); - // }); - // // test.skip("createTask with custom event and invocable triggers", () => { // const orderPlaced = defineEvent({ // name: "order.placed", @@ -347,88 +281,6 @@ describe("task event types", () => { // expect(task.name).toBe("process-order"); // }); // - // test.skip("createTask with database event and cron triggers", () => { - // const taskDef = defineTask({ - // name: "sync-contacts", - // }); - // - // const conductor = Conductor.create({ - // sql: {} as any, - // tasks: TaskSchemas.fromSchema([taskDef]), - // database: DatabaseSchema.fromGeneratedTypes(), - // context: {}, - // }); - // - // const task = conductor.createTask( - // { name: "sync-contacts" }, - // [ - // { cron: "0 * * * *", name: "hourly" }, - // { schema: "public", table: "contact", operation: "update" }, - // ], - // async (event, _ctx) => { - // // Event can be either cron or database event - // if (event.name === "hourly") { - // expectTypeOf(event).toEqualTypeOf<{ name: "hourly" }>(); - // } else if (event.name === "public.contact.update") { - // expectTypeOf(event.payload.tg_op).toEqualTypeOf<"UPDATE">(); - // // Both old and new should have values for update - // expectTypeOf(event.payload.old).not.toEqualTypeOf(); - // expectTypeOf(event.payload.new).not.toEqualTypeOf(); - // } - // }, - // ); - // - // expect(task.name).toBe("sync-contacts"); - // }); - // - // test.skip("createTask with all trigger types", () => { - // const paymentReceived = defineEvent({ - // name: "payment.received", - // payload: z.object({ paymentId: z.string(), amount: z.number() }), - // }); - // - // const taskDef = defineTask({ - // name: "audit-task", - // payload: z.object({ reason: z.string() }), - // }); - // - // const conductor = Conductor.create({ - // sql: {} as any, - // tasks: TaskSchemas.fromSchema([taskDef]), - // events: EventSchemas.fromSchema([paymentReceived]), - // database: DatabaseSchema.fromGeneratedTypes(), - // context: {}, - // }); - // - // const task = conductor.createTask( - // { name: "audit-task" }, - // [ - // { invocable: true }, - // { cron: "0 0 * * *", name: "daily" }, - // { event: "payment.received" }, - // { schema: "public", table: "contact", operation: "delete" }, - // ], - // async (event, _ctx) => { - // // Event can be any of the four types - // if (event.name === "pgconductor.invoke") { - // expectTypeOf(event.payload).toEqualTypeOf<{ reason: string }>(); - // } else if (event.name === "daily") { - // expectTypeOf(event).toEqualTypeOf<{ name: "daily" }>(); - // } else if (event.name === "payment.received") { - // expectTypeOf(event.payload.paymentId).toEqualTypeOf(); - // expectTypeOf(event.payload.amount).toEqualTypeOf(); - // } else if (event.name === "public.contact.delete") { - // expectTypeOf(event.payload.tg_op).toEqualTypeOf<"DELETE">(); - // expectTypeOf(event.payload.new).toEqualTypeOf(); - // // old should have the deleted row - // expectTypeOf(event.payload.old).not.toEqualTypeOf(); - // } - // }, - // ); - // - // expect(task.name).toBe("audit-task"); - // }); - // // test.skip("type error: custom event trigger without event definition", () => { // const taskDef = defineTask({ // name: "undefined-event-task", diff --git a/packages/pgconductor-js/tests/unit/worker-lifecycle.test.ts b/packages/pgconductor-js/tests/unit/worker-lifecycle.test.ts new file mode 100644 index 0000000..6da1797 --- /dev/null +++ b/packages/pgconductor-js/tests/unit/worker-lifecycle.test.ts @@ -0,0 +1,32 @@ +import { expect, test } from "bun:test"; +import { Worker } from "../../src/worker"; +import { DefaultLogger } from "../../src/lib/logger"; +import { MockDatabaseClient } from "../mocks/database-client.mock"; +import type { AnyTask } from "../../src/task"; + +const task = { + name: "lifecycle-task", + triggers: [{ invocable: true }], + maxAttempts: 3, + execute: async () => {}, +} as unknown as AnyTask; + +test("resets a worker after registration failure and permits retry", async () => { + let shouldFail = true; + const db = new MockDatabaseClient({ + registerWorker: async () => { + if (shouldFail) throw new Error("registration failed"); + }, + }); + const worker = new Worker("default", [task], db as never, new DefaultLogger()); + + await expect(worker.start("first-orchestrator")).rejects.toThrow("registration failed"); + // A failed startup is not a running worker: stopped must be already settled, + // rather than an unhandled rejected lifecycle promise. + await expect(worker.stopped).resolves.toBeUndefined(); + + shouldFail = false; + await worker.start("second-orchestrator"); + await worker.stop(); + await expect(worker.stopped).resolves.toBeUndefined(); +}); From 015e56158cc073e3124dceae273be3e75c12f34a Mon Sep 17 00:00:00 2001 From: psteinroe Date: Sat, 19 Sep 2026 20:00:29 +0000 Subject: [PATCH 2/3] refactor(events): normalize indexed event dispatch --- AGENTS.md | 388 +++++++- CLAUDE.md | 389 -------- docs/content/crafting-tasks/triggers.md | 32 +- docs/content/scaling/maintenance.md | 3 +- implementation-notes.md | 70 +- migrations/0000000001_setup.sql | 827 ++++++++++++------ .../pgconductor-js/src/database-client.ts | 32 +- .../pgconductor-js/src/event-definition.ts | 20 +- .../src/event-trigger-validation.ts | 172 +++- packages/pgconductor-js/src/generated/sql.ts | 827 ++++++++++++------ .../pgconductor-js/src/maintenance-task.ts | 11 - packages/pgconductor-js/src/query-builder.ts | 38 +- packages/pgconductor-js/src/worker.ts | 4 +- .../tests/integration/event-pipeline.test.ts | 629 ++++++++----- .../tests/integration/event-triggers.test.ts | 8 +- .../subscription-lifecycle.test.ts | 443 +++++----- .../tests/mocks/database-client.mock.ts | 1 - .../tests/mocks/in-memory-database-client.ts | 357 ++++---- .../tests/unit/event-pipeline-types.test.ts | 4 +- .../tests/unit/event-trigger-types.test.ts | 55 ++ .../unit/event-trigger-validation.test.ts | 45 + .../unit/in-memory-database-client.test.ts | 86 +- 22 files changed, 2775 insertions(+), 1666 deletions(-) delete mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md index 0f445b3..295ee63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,389 @@ # pgconductor -Read @CLAUDE.md +A durable task execution system built on PostgreSQL. + +> **Important**: Keep this documentation up-to-date. When making architectural changes, refactoring core components, or introducing new patterns worth mentioning, update this file accordingly. + +## Project Structure + +``` +pgconductor/ +├── justfile # Command runner (use `just` for all commands) +├── migrations/ +│ └── 0000000001_setup.sql # Core schema, tables, SQL functions +└── packages/ + └── pgconductor-js/ + ├── src/ + │ ├── conductor.ts # Task registry and invocation entry point + │ ├── orchestrator.ts # Worker lifecycle manager + │ ├── worker.ts # Fetch→execute→flush pipeline + │ ├── task.ts # Task wrapper with execute method + │ ├── task-definition.ts # Zod-based task definitions + │ ├── task-context.ts # Context API (step, sleep, invoke) + │ ├── database-client.ts # PostgreSQL client wrapper + │ ├── lib/ # Utilities (deferred, async-queue, etc.) + │ └── generated/ + │ └── sql.ts # Auto-generated types from SQL + └── tests/ + ├── unit/ # Unit tests for utilities (no DB) + ├── integration/ # End-to-end tests with real DB + └── fixtures/ # Test utilities (TestDatabasePool) +``` + +## Development Workflow + +### Command Runner + +Use `just` for all project commands (defined in `justfile`): + +```bash +just build-migrations # Rebuild TypeScript types from migrations +``` + +### Running Tests + +Tests use Bun test runner. **Always run typecheck with tests:** + +```bash +bun test && bun run typecheck # Run tests and type checking (ALWAYS) +bun test # All tests (unit + integration) +bun test tests/unit/ # Unit tests only +bun test tests/integration/ # Integration tests only +``` + +**Unit Tests** (`tests/unit/`) +- Test utility functions in isolation (no database required) +- Examples: `deferred.test.ts`, `map-concurrent.test.ts`, `async-queue.test.ts`, `wait-for.test.ts` +- Fast, no setup required + +```typescript +import { test, expect } from "bun:test"; +import { Deferred } from "../../../src/lib/deferred"; + +test("deferred resolves", async () => { + const deferred = new Deferred(); + deferred.resolve(42); + expect(await deferred.promise).toBe(42); +}); +``` + +**Integration Tests** (`tests/integration/`) +- Test end-to-end workflows with real PostgreSQL database +- Examples: `basic-execution.test.ts`, `step-support.test.ts`, `invoke-support.test.ts` +- Use `TestDatabasePool` fixture for isolated database instances + +```typescript +import { TestDatabasePool } from "../fixtures/test-database"; + +let pool: TestDatabasePool; + +beforeAll(async () => { + pool = await TestDatabasePool.create(); +}, 60000); + +afterAll(async () => { + await pool?.destroy(); +}); + +test("example", async () => { + const db = await pool.child(); // Isolated connection + + // Create conductor and task + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([taskDefinition]), + context: {}, + }); + + conductor.createTask( + { name: "example-task" }, + { invocable: true }, + async (event, _ctx) => { /* handler */ }, + ); + + // Initialize schema before invoking tasks + await conductor.ensureInstalled(); + + // Now safe to invoke tasks + await conductor.invoke({ name: "example-task" }, {}); +}); +``` + +**Important**: Integration tests must call `await conductor.ensureInstalled()` before invoking tasks. This initializes the database schema (tables, functions, etc.). Without this, `conductor.invoke()` will fail with "schema pgconductor does not exist". + +### Modifying Migrations + +1. Edit `migrations/0000000001_setup.sql` directly (in-place) +2. Run `just build-migrations` to regenerate types +3. Run tests to verify changes + +**Note**: This project is under active development. All SQL changes should be made **in-place** by editing the existing migration file, not by creating new migration files. + +The migration file contains: +- Table schemas (`tasks`, `executions`, `steps`, `failed_executions`, `test_config`) +- SQL functions (all logic is in SQL, not application code) +- Indexes + +**Note**: The `test_config` table is used for testing purposes (e.g., controlling time with `fake_now`). + +## Architecture + +### Core Components + +**Worker** (`worker.ts`) +- Implements async pipeline: fetch → execute → flush +- Polls database for ready executions using `get_executions()` +- Executes tasks with concurrency control (via `mapConcurrent`) +- Batches and flushes results back to database +- Handles graceful shutdown via AbortController + +**TaskContext** (`task-context.ts`) +- Provides API to task functions: `step()`, `sleep()`, `invoke()` +- All operations are idempotent (use steps as memoization) +- Hangup pattern: abort execution and return never-resolving promise +- Resume happens automatically when database wakes execution + +**Conductor** (`conductor.ts`) +- Task registry and factory +- Entry point for invoking tasks +- Manages database client lifecycle + +**Orchestrator** (`orchestrator.ts`) +- Manages multiple workers +- Handles startup/shutdown coordination +- Provides `stopped` promise for graceful shutdown + +**DatabaseClient** (`database-client.ts`) +- Wraps all SQL function calls for easier unit testing and mocking +- All SQL functions are called through this interface +- Example: `db.getExecutions()`, `db.returnExecutions()`, `db.invoke()` + +### SQL Functions + +**Important**: All core logic lives in PostgreSQL functions (not application code). The TypeScript layer is intentionally thin - it only orchestrates calls to SQL functions via `DatabaseClient`. + +SQL functions in `migrations/0000000001_setup.sql`: +- `get_executions()`: Fetch and claim ready executions +- `return_executions()`: Process results (completed/failed/released) +- `invoke()`: Create child execution and set parent waiting state +- `backoff_seconds()`: Return retry delay based on attempt number +- Helper functions: `current_time()`, `portable_uuidv7()`, etc. + +### Key Design Patterns + +**Hangup/Resume** +- When a task calls `ctx.sleep()` or `ctx.invoke()`, the worker aborts the task +- The execution remains in the database with updated `run_at` or `waiting_on_execution_id` +- Worker polls and resumes execution when ready +- Steps provide memoization across hangups + +**Step Memoization** +- `ctx.step(name, fn)` checks if step exists in database before executing +- If exists, returns cached result +- If not, executes fn and saves result +- This enables idempotent retries and resume after hangup + +**Cascade Failures** +- When child fails permanently (attempts >= max_attempts), parent is moved to `failed_executions` +- Implemented in `return_executions()` via `permanently_failed_children` CTE +- Parent receives error like "Child execution failed: " + +**Infinity Pattern** +- PostgreSQL `'infinity'::timestamptz` for indefinite waiting +- Used when `invoke()` called without timeout +- Parent waits forever until child completes + +### Task Configuration Options + +Tasks accept a configuration object with these options: + +```typescript +conductor.createTask( + { + name: "my-task", + queue: "default", // Queue name (default: "default") + maxAttempts: 3, // Max retry attempts before permanent failure (default: from DB) + flushInterval: 2000, // How often to flush results to DB in ms (default: 2000) + pollInterval: 1000, // How often to poll for new executions in ms (default: 1000) + partition: false, // Enable partitioning (default: false) + window: ["09:00", "17:00"], // Time window for execution [start, end] + }, + handler, +); +``` + +**Note**: Lower `pollInterval` values (e.g., 100ms) are useful in tests for faster execution cycles. + +## Query Optimization Principles + +Guidelines for SQL functions: + +1. **Filter before joining**: Apply WHERE on small result sets before joining large tables + ```sql + -- Good: filter results first (0-1 rows), then join to find parent + FROM results r + WHERE r.status = 'completed' + JOIN pgconductor.executions parent_e ON parent_e.waiting_on_execution_id = r.execution_id + ``` + +2. **Materialize expensive operations**: Use CTEs to compute once and reuse + ```sql + permanently_failed_children AS ( + SELECT r.execution_id, r.error + FROM results r + JOIN pgconductor.executions e ON e.id = r.execution_id + JOIN pgconductor.tasks w ON w.key = e.task_key + WHERE r.status = 'failed' AND e.attempts >= w.max_attempts + ) + -- Reuse in multiple places without recomputing join + ``` + +3. **Prefer SQL functions with CTEs over plpgsql**: CTEs are declarative and easier to optimize + +4. **No foreign keys**: Performance optimization - rely on application/SQL function logic + +## Common Development Tasks + +### Adding New Context Method + +1. Add method to `TaskContext` class in `task-context.ts` +2. Implement using steps/database operations +3. Add integration test in `tests/integration/` +4. Update documentation + +### Adding New SQL Function + +1. Add function to `migrations/0000000001_setup.sql` +2. Run `just build-migrations` to regenerate types +3. Update `database-client.ts` if wrapper needed +4. Add tests + +### Debugging Test Failures + +Common issues: +- **Timing issues**: Increase wait times (backoff schedule is 15s, 30s, 60s...) +- **Cascade failures**: Check `permanently_failed_children` CTE logic +- **Infinity serialization**: postgres.js serializes infinity as null in JSON + +Use `console.log()` in tests to inspect database state: +```typescript +const rows = await db.sql`SELECT * FROM pgconductor.executions`; +console.log(JSON.stringify(rows, null, 2)); +``` + +### Controlling Time in Tests + +The `pgconductor.current_time()` function checks `current_setting('pgconductor.fake_now')` before returning the real time. This allows tests to control time without waiting: + +```typescript +// Test databases use max: 1 connection pool (see TestDatabase.create()) +// This ensures all queries share the same connection and see the same fake time + +// Set fake time before starting orchestrator +const fakeTime = new Date("2024-01-01T12:00:00Z"); +await db.client.setFakeTime(fakeTime); + +// Now all operations use fake time +await db.sql`SELECT pgconductor.current_time()`; // Returns 2024-01-01 12:00:00 + +// Advance time by 1 hour (works immediately with session-level SET) +const laterTime = new Date(fakeTime.getTime() + 3600000); +await db.client.setFakeTime(laterTime); + +// Reset to real time +await db.client.clearFakeTime(); +``` + +**How it works:** +- Each test creates an isolated database via `pool.child()` +- The database connection uses `max: 1` pool size +- All queries (test, conductor, worker) share the same connection +- `setFakeTime()` uses session-level `SET pgconductor.fake_now` +- Since everyone shares the connection, everyone sees the same time + +This is particularly useful for testing: +- Sleep/delayed executions +- Timeouts +- Backoff schedules +- Time-based windows + +**Note**: Always clean up fake_now at the end of tests to avoid affecting other tests. + +--- + +## Planned Changes + +### Queue System (In Progress) + +See **QUEUE-IMPLEMENTATION-PLAN.md** for detailed implementation plan. + +**Goal**: Transition from one-worker-per-task to queue-based worker model with explicit partitioning and batch processing. + +**Key Changes**: +- Tasks will have a `queue` field (defaults to "default") +- Workers operate on queues (handling multiple tasks) +- Explicit `createWorker(queueName, settings, tasks)` API +- Auto-provisioned default queue with zero config +- Task-level batching support + +**Status**: Phase 1 complete (type system updates) ✅ + +--- + +## Development Environment + +### Bun Runtime + +This project uses Bun for running TypeScript, tests, and package management. + +```bash +# Package management +bun install # Install dependencies + +# Running code +bun # Run TypeScript directly +bun run